Skip to content

Commit 5526c37

Browse files
korowakorowa
committed
AmateurMergeTree: ci
1 parent fecda06 commit 5526c37

4 files changed

Lines changed: 276 additions & 0 deletions

File tree

‎.github/workflows/amateur_ci.yaml‎

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
---
2+
name: AmateurCI
3+
4+
on:
5+
push:
6+
branches: ['amateur-merge-tree']
7+
8+
jobs:
9+
build:
10+
runs-on: ubuntu-latest
11+
name: "Build"
12+
steps:
13+
- name: Checkout code
14+
uses: actions/checkout@v6
15+
with:
16+
submodules: recursive
17+
18+
- name: Restore ccache
19+
uses: actions/cache/restore@v4
20+
with:
21+
path: /tmp/ccache
22+
key: ccache-${{ github.sha }}
23+
restore-keys: |
24+
ccache-
25+
26+
- name: Build
27+
run: |
28+
docker run --rm -v $PWD:/ClickHouse -v /tmp/ccache:/tmp/ccache clickhouse/fasttest:b63954f8a1b316d0686e \
29+
/bin/bash -c "
30+
set -e
31+
apt-get update && apt-get install -y ccache
32+
export CC=clang-21
33+
export CXX=clang++-21
34+
export CCACHE_DIR=/tmp/ccache
35+
cd /ClickHouse/
36+
cmake -DCOMPILER_CACHE=ccache -DCMAKE_BUILD_TYPE=Debug -DOMIT_HEAVY_DEBUG_SYMBOLS=1 -DENABLE_LIBRARIES=0 -DENABLE_UTILS=0 -DENABLE_TESTS=0 -DENABLE_RUST=0 -DENABLE_JEMALLOC=1 -DENABLE_YAML_CPP=1 -DENABLE_AWS_S3=1 -G Ninja -S . -B build
37+
ninja -C build
38+
ccache --show-stats
39+
"
40+
41+
- name: Save ccache
42+
if: github.ref_name == 'amateur-merge-tree'
43+
uses: actions/cache/save@v4
44+
with:
45+
path: /tmp/ccache
46+
key: ccache-${{ github.sha }}
47+
48+
- name: Integration tests
49+
run: |
50+
docker run --rm -v /var/run/docker.sock:/var/run/docker.sock -v $PWD:$PWD --network host clickhouse/integration-tests-runner:c909f2e80ec78b3a6532 \
51+
/bin/bash -c "
52+
CLICKHOUSE_TESTS_SERVER_BIN_PATH=$PWD/build/programs/clickhouse \
53+
pytest $PWD/tests/integration/test_amateur_merge_tree/ -v --timeout=600
54+
"

‎tests/integration/test_amateur_merge_tree/__init__.py‎

Whitespace-only changes.

‎tests/integration/test_amateur_merge_tree/configs/config.yaml‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
---
2+
storage_configuration:
3+
disks:
4+
s3:
5+
type: object_storage
6+
object_storage_type: s3
7+
metadata_type: amateur
8+
endpoint: http://minio1:9001/root/data/
9+
keeper_prefix: /clickhouse/disks/s3
10+
access_key_id: minio
11+
secret_access_key: ClickHouse_Minio_P@ssw0rd
12+
policies:
13+
s3:
14+
volumes:
15+
- main:
16+
disks:
17+
- s3

‎tests/integration/test_amateur_merge_tree/test.py‎

Lines changed: 205 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,205 @@
1+
#!/usr/bin/env python3
2+
3+
import time
4+
5+
import pytest
6+
7+
from helpers.cluster import ClickHouseCluster
8+
from helpers.test_tools import TSV
9+
10+
cluster = ClickHouseCluster(__file__)
11+
12+
13+
def wait_table_sync(zk_client, zk_table_path):
14+
_, parts_stat = zk_client.get(f"{zk_table_path}/parts")
15+
nodes = zk_client.get_children(f"{zk_table_path}/replicas")
16+
17+
for _ in range(60):
18+
nodes_in_sync = 0
19+
for node in nodes:
20+
node_epoch, _ = zk_client.get(f"{zk_table_path}/replicas/{node}/epoch")
21+
if int(node_epoch) >= parts_stat.pzxid:
22+
nodes_in_sync += 1
23+
24+
if nodes_in_sync == len(nodes):
25+
return
26+
27+
time.sleep(0.5)
28+
29+
30+
@pytest.fixture(scope="module")
31+
def started_cluster():
32+
try:
33+
cluster.add_instance(
34+
"node1",
35+
main_configs=["configs/config.yaml"],
36+
with_minio=True,
37+
with_zookeeper=True,
38+
stay_alive=True,
39+
use_keeper=False,
40+
cpu_limit=3,
41+
)
42+
cluster.add_instance(
43+
"node2",
44+
main_configs=["configs/config.yaml"],
45+
with_minio=True,
46+
with_zookeeper=True,
47+
stay_alive=True,
48+
use_keeper=False,
49+
cpu_limit=3
50+
)
51+
cluster.start()
52+
yield cluster
53+
finally:
54+
cluster.shutdown()
55+
56+
57+
def test_insert_select(started_cluster):
58+
"""
59+
Basic scenario with inserts and replica synchronization
60+
"""
61+
62+
for node_name, node in cluster.instances.items():
63+
data = node.query(f"""
64+
CREATE TABLE test_basic (key UInt32, val String)
65+
ENGINE = AmateurMergeTree('/clickhouse/tables/default/test_basic', '{node_name}')
66+
ORDER BY key
67+
SETTINGS storage_policy = 's3'
68+
""")
69+
70+
for i in range(10):
71+
node = list(cluster.instances.values())[i % 2]
72+
node.query(f"""
73+
INSERT INTO test_basic SELECT number + {i} * 10 as key, 'val-' || key AS val FROM numbers(10)
74+
""")
75+
76+
zk_client = cluster.get_kazoo_client("zoo1")
77+
wait_table_sync(zk_client, "/clickhouse/tables/default/test_basic")
78+
79+
expected = [[100, 0, 'val-0', 99, 'val-99']]
80+
for node_name, node in cluster.instances.items():
81+
actual = node.query("""
82+
SELECT count(), min(key), argMin(val, key), max(key), argMax(val, key) FROM test_basic
83+
""")
84+
85+
assert TSV(actual) == TSV(expected), f"Received an unexpected result from node {node_name}"
86+
87+
88+
def test_drop_partititon(started_cluster):
89+
"""
90+
Insert into partitioned tables and drop partition
91+
"""
92+
93+
for node_name, node in cluster.instances.items():
94+
data = node.query(f"""
95+
CREATE TABLE test_partitioned (key UInt32, prt UInt32, val String)
96+
ENGINE = AmateurMergeTree('/clickhouse/tables/default/test_partitioned', '{node_name}')
97+
PARTITION BY prt
98+
ORDER BY key
99+
SETTINGS storage_policy = 's3'
100+
""")
101+
102+
for i in range(10):
103+
node = list(cluster.instances.values())[i % 2]
104+
node.query(f"""
105+
INSERT INTO test_partitioned SELECT number + {i} * 10 as key, number % 5 AS prt, 'val-' || key FROM numbers(10)
106+
""")
107+
108+
cluster.instances["node1"].query("ALTER TABLE test_partitioned DROP PARTITION 1")
109+
cluster.instances["node2"].query("ALTER TABLE test_partitioned DROP PARTITION 3")
110+
111+
zk_client = cluster.get_kazoo_client("zoo1")
112+
wait_table_sync(zk_client, "/clickhouse/tables/default/test_partitioned")
113+
114+
expected = [[60, 0, 'val-0', 99, 'val-99', 0, 0]]
115+
for node_name, node in cluster.instances.items():
116+
actual = node.query("""
117+
SELECT count(), min(key), argMin(val, key), max(key), argMax(val, key),
118+
countIf(prt = 1), countIf(prt = 3)
119+
FROM test_partitioned
120+
""")
121+
122+
assert TSV(actual) == TSV(expected), f"Received an unexpected result from node {node_name}"
123+
124+
125+
def test_optimize(started_cluster):
126+
"""
127+
Insert into partitioned tables and drop partition
128+
"""
129+
130+
for node_name, node in cluster.instances.items():
131+
data = node.query(f"""
132+
CREATE TABLE test_optimize(key UInt32, val String)
133+
ENGINE = AmateurMergeTree('/clickhouse/tables/default/test_optimize', '{node_name}')
134+
ORDER BY key
135+
SETTINGS storage_policy = 's3'
136+
""")
137+
138+
for i in range(10):
139+
node = list(cluster.instances.values())[i % 2]
140+
node.query(f"""
141+
INSERT INTO test_optimize SELECT number + {i} * 10 as key, 'val-' || key FROM numbers(10)
142+
""")
143+
144+
zk_client = cluster.get_kazoo_client("zoo1")
145+
wait_table_sync(zk_client, "/clickhouse/tables/default/test_optimize")
146+
147+
for node_name, node in cluster.instances.items():
148+
actual = node.query("OPTIMIZE TABLE test_optimize FINAL")
149+
150+
wait_table_sync(zk_client, "/clickhouse/tables/default/test_optimize")
151+
152+
expected = [[60, 0, 'val-0', 99, 'val-99', 0, 0, 'all_0_9']]
153+
for node_name, node in cluster.instances.items():
154+
actual = node.query("""
155+
SELECT count(), min(key), argMin(val, key), max(key), argMax(val, key),
156+
any(substr(_part, 1, 7))
157+
FROM test_optimize
158+
""")
159+
160+
161+
def test_add_remove_replica(started_cluster):
162+
"""
163+
Add table replica while the data already exists
164+
"""
165+
166+
cluster.instances["node1"].query("""
167+
CREATE TABLE test_add_remove_replica (key UInt32, val String)
168+
ENGINE = AmateurMergeTree('/clickhouse/tables/default/test_add_remove_replica', 'node1')
169+
ORDER BY key
170+
SETTINGS storage_policy = 's3'
171+
""")
172+
173+
for i in range(10):
174+
cluster.instances["node1"].query(f"""
175+
INSERT INTO test_add_remove_replica SELECT number + {i} * 10 as key, 'val-' || key AS val FROM numbers(10)
176+
""")
177+
178+
expected = [[100, 0, 'val-0', 99, 'val-99']]
179+
180+
actual_first = cluster.instances["node1"].query("""
181+
SELECT count(), min(key), argMin(val, key), max(key), argMax(val, key) FROM test_add_remove_replica
182+
""")
183+
assert TSV(actual_first) == TSV(expected), "Received an unexpected result from node1"
184+
185+
cluster.instances["node2"].query("""
186+
CREATE TABLE test_add_remove_replica (key UInt32, val String)
187+
ENGINE = AmateurMergeTree('/clickhouse/tables/default/test_add_remove_replica', 'node2')
188+
ORDER BY key
189+
SETTINGS storage_policy = 's3'
190+
""")
191+
192+
zk_client = cluster.get_kazoo_client("zoo1")
193+
wait_table_sync(zk_client, "/clickhouse/tables/default/test_add_remove_replica")
194+
195+
actual_replica = cluster.instances["node2"].query("""
196+
SELECT count(), min(key), argMin(val, key), max(key), argMax(val, key) FROM test_add_remove_replica
197+
""")
198+
assert TSV(actual_first) == TSV(expected), "Received an unexpected result from node2"
199+
200+
cluster.instances["node2"].query("DROP TABLE test_add_remove_replica SYNC")
201+
202+
actual_first = cluster.instances["node1"].query("""
203+
SELECT count(), min(key), argMin(val, key), max(key), argMax(val, key) FROM test_add_remove_replica
204+
""")
205+
assert TSV(actual_first) == TSV(expected), "Received an unexpected result from node1 after replica drop"

0 commit comments

Comments
 (0)