Skip to content

Commit ecf2d18

Browse files
MyroTkMyroTk
authored
Merge pull request #1374 from Altinity/builds/release_cicd/add_symtable_bin
Release CI/CD: Automated support for binary with symtable
2 parents 7981545 + d680c22 commit ecf2d18

File tree

4 files changed

+129
-16
lines changed

4 files changed

+129
-16
lines changed

‎.github/workflows/docker_publish.yml‎

Lines changed: 34 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -69,9 +69,42 @@ jobs:
6969
echo "Pulling the image"
7070
docker pull $IMAGE
7171
72+
# Create container for version and symtable checks
73+
echo "Creating container for checks..."
74+
CONTAINER_ID=$(docker create $IMAGE $COMPONENT --version)
75+
76+
# Check if binary has symbol table (not stripped)
77+
echo "Checking if binary has symtable..."
78+
# Try common binary locations
79+
BINARY_PATH=""
80+
for path in "/usr/bin/clickhouse" "/usr/bin/$COMPONENT"; do
81+
if docker cp "$CONTAINER_ID:$path" /tmp/clickhouse-check 2>/dev/null; then
82+
BINARY_PATH="$path"
83+
break
84+
fi
85+
done
86+
87+
if [ -z "$BINARY_PATH" ]; then
88+
echo "✗ Could not find clickhouse binary in container"
89+
docker rm "$CONTAINER_ID"
90+
exit 1
91+
fi
92+
93+
echo "Found binary at $BINARY_PATH"
94+
if readelf -S /tmp/clickhouse-check | grep -q '\.symtab'; then
95+
echo "✓ Binary has symtable"
96+
else
97+
echo "✗ Binary is missing symtable"
98+
docker rm "$CONTAINER_ID"
99+
rm -f /tmp/clickhouse-check
100+
exit 1
101+
fi
102+
rm -f /tmp/clickhouse-check
103+
72104
# Get version and clean it up
73105
echo "Getting version from image..."
74-
VERSION_OUTPUT=$(docker run --rm $IMAGE $COMPONENT --version)
106+
VERSION_OUTPUT=$(docker start -a "$CONTAINER_ID")
107+
docker rm "$CONTAINER_ID"
75108
echo "Raw version output: $VERSION_OUTPUT"
76109
77110
# Extract just the version number

‎.github/workflows/sign_and_release.yml‎

Lines changed: 73 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -102,8 +102,31 @@ jobs:
102102
echo "Running clickhouse binary..."
103103
./clickhouse -q'q'
104104
105-
echo "Stripping the binary..."
106-
strip clickhouse -o clickhouse-stripped
105+
echo "Downloading clickhouse-stripped binary..."
106+
if ! aws s3 cp "${SRC_URL}/${ARM_PATH}/clickhouse-stripped" clickhouse-stripped; then
107+
echo "Failed to download clickhouse-stripped binary"
108+
exit 1
109+
fi
110+
111+
chmod +x clickhouse-stripped
112+
113+
echo "Running clickhouse-stripped binary..."
114+
./clickhouse-stripped -q'q'
115+
116+
echo "Downloading clickhouse-common-static tar..."
117+
if ! aws s3 cp "${SRC_URL}/${ARM_PATH}/clickhouse-common-static-${{ inputs.package_version }}-arm64.tgz" clickhouse-common-static.tgz; then
118+
echo "Failed to download clickhouse-common-static tar"
119+
exit 1
120+
fi
121+
122+
tar -xvf clickhouse-common-static.tgz
123+
124+
if readelf -S "clickhouse-common-static-${{ inputs.package_version }}/usr/bin/clickhouse" | grep -q '\.symtab'; then
125+
echo "✓ Binary has symtable"
126+
else
127+
echo "✗ Binary is missing symtable"
128+
exit 1
129+
fi
107130
108131
echo "Uploading processed binaries..."
109132
if ! aws s3 cp clickhouse "${SRC_URL}/${ARM_PATH}/arm64-bin/non-self-extracting/"; then
@@ -114,6 +137,10 @@ jobs:
114137
echo "Failed to upload stripped clickhouse binary"
115138
exit 1
116139
fi
140+
if ! aws s3 cp "clickhouse-common-static-${{ inputs.package_version }}/usr/bin/clickhouse" "${SRC_URL}/${ARM_PATH}/arm64-bin/non-self-extracting/clickhouse-with-symtable"; then
141+
echo "Failed to upload clickhouse-with-symtable binary"
142+
exit 1
143+
fi
117144
118145
copy-packages:
119146
needs: extract-package-info
@@ -177,8 +204,32 @@ jobs:
177204
echo "Running clickhouse binary..."
178205
./clickhouse -q'q'
179206
180-
echo "Stripping the binary..."
181-
strip clickhouse -o clickhouse-stripped
207+
echo "Downloading clickhouse-stripped binary..."
208+
if ! aws s3 cp "${DEST_URL}/packages/AMD_PACKAGES/amd64-bin/clickhouse-stripped" clickhouse-stripped; then
209+
echo "Failed to download clickhouse-stripped binary"
210+
exit 1
211+
fi
212+
213+
chmod +x clickhouse-stripped
214+
215+
echo "Running clickhouse-stripped binary..."
216+
./clickhouse-stripped -q'q'
217+
218+
219+
echo "Downloading clickhouse-common-static tar..."
220+
if ! aws s3 cp "${DEST_URL}/packages/AMD_PACKAGES/clickhouse-common-static-${{ inputs.package_version }}-amd64.tgz" clickhouse-common-static.tgz; then
221+
echo "Failed to download clickhouse-common-static tar"
222+
exit 1
223+
fi
224+
225+
tar -xvf clickhouse-common-static.tgz
226+
227+
if readelf -S "clickhouse-common-static-${{ inputs.package_version }}/usr/bin/clickhouse" | grep -q '\.symtab'; then
228+
echo "✓ Binary has symtable"
229+
else
230+
echo "✗ Binary is missing symtable"
231+
exit 1
232+
fi
182233
183234
echo "Uploading processed binaries..."
184235
if ! aws s3 cp clickhouse "${DEST_URL}/packages/AMD_PACKAGES/amd64-bin/non-self-extracting/"; then
@@ -189,6 +240,10 @@ jobs:
189240
echo "Failed to upload stripped clickhouse binary"
190241
exit 1
191242
fi
243+
if ! aws s3 cp clickhouse-common-static-${{ inputs.package_version }}/usr/bin/clickhouse "${DEST_URL}/packages/AMD_PACKAGES/amd64-bin/non-self-extracting/clickhouse-with-symtable"; then
244+
echo "Failed to upload stripped clickhouse binary"
245+
exit 1
246+
fi
192247
193248
copy-test-results:
194249
needs: extract-package-info
@@ -259,6 +314,20 @@ jobs:
259314
repository: Altinity/ClickHouse
260315
path: ClickHouse
261316

317+
- name: Install required packages
318+
run: |
319+
echo "Installing required packages..."
320+
sudo add-apt-repository -y universe
321+
sudo apt-get update
322+
sudo apt-get install -y software-properties-common python3-pip apt-utils gnupg rpm createrepo-c file wget
323+
wget -q http://archive.ubuntu.com/ubuntu/pool/universe/d/dpkg-sig/dpkg-sig_0.13.1+nmu4_all.deb
324+
sudo dpkg -i dpkg-sig_0.13.1+nmu4_all.deb || sudo apt-get install -f -y
325+
rm dpkg-sig_0.13.1+nmu4_all.deb
326+
sudo apt-add-repository --yes --update ppa:ansible/ansible
327+
sudo apt-get install -y ansible
328+
sudo ln -s /usr/bin/createrepo_c /usr/bin/createrepo
329+
pip3 install boto3 botocore natsort --break-system-packages
330+
262331
- name: Download packages
263332
run: |
264333
if ! aws s3 sync "${DEST_URL}/packages/ARM_PACKAGES/" /home/runner/.cache/tmp/packages --exact-timestamps; then
@@ -273,16 +342,6 @@ jobs:
273342
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
274343
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
275344

276-
- name: Install required packages
277-
run: |
278-
echo "Installing required packages..."
279-
sudo apt-get update
280-
sudo apt-get install -y software-properties-common python3-pip dpkg-sig apt-utils gnupg rpm createrepo-c file
281-
sudo apt-add-repository --yes --update ppa:ansible/ansible
282-
sudo apt-get install -y ansible
283-
sudo ln -s /usr/bin/createrepo_c /usr/bin/createrepo
284-
pip3 install boto3 botocore
285-
286345
- name: Set up GPG passphrase
287346
run: |
288347
if [ "${RELEASE_ENVIRONMENT}" == "production" ]; then

‎tests/ci/release/packaging/ansible/roles/update_bin_repo/tasks/main.yml‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@
1515
mv /home/runner/.cache/tmp/packages/{{ item }}-bin/clickhouse-stripped {{ local_repo_path }}/{{ repo_prefix }}bin-repo/{{ item }}/v{{ pkgver }}/self-extracting/clickhouse-stripped
1616
mv /home/runner/.cache/tmp/packages/{{ item }}-bin/non-self-extracting/clickhouse {{ local_repo_path }}/{{ repo_prefix }}bin-repo/{{ item }}/v{{ pkgver }}/non-self-extracting/clickhouse
1717
mv /home/runner/.cache/tmp/packages/{{ item }}-bin/non-self-extracting/clickhouse-stripped {{ local_repo_path }}/{{ repo_prefix }}bin-repo/{{ item }}/v{{ pkgver }}/non-self-extracting/clickhouse-stripped
18+
mv /home/runner/.cache/tmp/packages/{{ item }}-bin/non-self-extracting/clickhouse-with-symtable {{ local_repo_path }}/{{ repo_prefix }}bin-repo/{{ item }}/v{{ pkgver }}/non-self-extracting/clickhouse-with-symtable
1819
loop:
1920
- amd64
2021
- arm64
@@ -27,10 +28,12 @@
2728
- amd64/v{{ pkgver }}/self-extracting/clickhouse-stripped
2829
- amd64/v{{ pkgver }}/non-self-extracting/clickhouse
2930
- amd64/v{{ pkgver }}/non-self-extracting/clickhouse-stripped
31+
- amd64/v{{ pkgver }}/non-self-extracting/clickhouse-with-symtable
3032
- arm64/v{{ pkgver }}/self-extracting/clickhouse
3133
- arm64/v{{ pkgver }}/self-extracting/clickhouse-stripped
3234
- arm64/v{{ pkgver }}/non-self-extracting/clickhouse
3335
- arm64/v{{ pkgver }}/non-self-extracting/clickhouse-stripped
36+
- arm64/v{{ pkgver }}/non-self-extracting/clickhouse-with-symtable
3437
when: (pkgver | regex_replace('^(\\d+).*$', '\\1')) is version('24', '>=')
3538

3639
- name: Copy new binaries

‎tests/ci/release/packaging/dirindex/dirindexgen.py‎

Lines changed: 19 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,10 @@
11
#!/bin/env python3
22

33
import argparse
4+
import re
45
import textwrap
56
import boto3
7+
from natsort import natsorted
68

79

810
def folder_list_add(folders, key, value):
@@ -39,6 +41,16 @@ def folderjoin(items, slashes=False):
3941
return result
4042

4143

44+
def is_bin_repo_arch_folder(folder):
45+
"""
46+
Check if the folder is a bin-repo architecture folder that should use natural sorting.
47+
Matches patterns like: bin-repo/arm64, antalya-bin-repo/amd64, fips-bin-repo/arm64, etc.
48+
"""
49+
# Match folders ending with (bin-repo|antalya-bin-repo|fips-bin-repo|hotfix-bin-repo)/(arm64|amd64)
50+
pattern = r'(bin-repo|antalya-bin-repo|fips-bin-repo|hotfix-bin-repo)/(arm64|amd64)$'
51+
return bool(re.search(pattern, folder))
52+
53+
4254
def main():
4355
parser = argparse.ArgumentParser()
4456
parser.add_argument(dest='fqdn', help="Name of S3 bucket and domain name")
@@ -98,7 +110,13 @@ def main():
98110
parent_folder = folderjoin(folder.split("/")[:-1], slashes=True)
99111
indexdata.append(f'<a href="{parent_folder}">{parent_dir_str}</a>' +
100112
f'{" ":{maxlen-len(parent_dir_str)}} {"-":20} {"-":20}\n')
101-
for sub in subs:
113+
# Apply natural sorting (descending) for bin-repo architecture folders
114+
# to show newest versions first (e.g., v25.3 before v24.8 before v23.8)
115+
if is_bin_repo_arch_folder(folder):
116+
sorted_subs = list(reversed(natsorted(subs)))
117+
else:
118+
sorted_subs = sorted(subs)
119+
for sub in sorted_subs:
102120
sub_path = folderjoin([folder, sub], slashes=True)
103121
indexdata.append(f'<a href="{sub_path}">{sub}/</a>{" ":{maxlen-len(sub)-1}}' +
104122
f' {"-":20} {"-":20}\n')

0 commit comments

Comments
 (0)