AmateurMergeTree: ci

Author: korowa

.github/workflows/amateur_ci.yaml

@@ -0,0 +1,54 @@
+---
+name: AmateurCI
+
+on:
+  push:
+    branches: ['amateur-merge-tree']
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    name: "Build"
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v6
+        with:
+          submodules: recursive
+
+      - name: Restore ccache
+        uses: actions/cache/restore@v4
+        with:
+          path: /tmp/ccache
+          key: ccache-${{ github.sha }}
+          restore-keys: |
+            ccache-
+
+      - name: Build
+        run: |
+          docker run --rm -v $PWD:/ClickHouse -v /tmp/ccache:/tmp/ccache clickhouse/fasttest:b63954f8a1b316d0686e \
+            /bin/bash -c "
+              set -e
+              apt-get update && apt-get install -y ccache
+              export CC=clang-21
+              export CXX=clang++-21
+              export CCACHE_DIR=/tmp/ccache
+              cd /ClickHouse/
+              cmake -DCOMPILER_CACHE=ccache -DCMAKE_BUILD_TYPE=Debug -DOMIT_HEAVY_DEBUG_SYMBOLS=1 -DENABLE_LIBRARIES=0 -DENABLE_UTILS=0 -DENABLE_TESTS=0 -DENABLE_RUST=0 -DENABLE_JEMALLOC=1 -DENABLE_YAML_CPP=1 -DENABLE_AWS_S3=1 -G Ninja -S . -B build
+              ninja -C build
+              ccache --show-stats
+            "
+
+      - name: Save ccache
+        if: github.ref_name == 'amateur-merge-tree'
+        uses: actions/cache/save@v4
+        with:
+          path: /tmp/ccache
+          key: ccache-${{ github.sha }}
+
+      - name: Integration tests
+        run: |
+          docker run --rm -v /var/run/docker.sock:/var/run/docker.sock -v $PWD:$PWD --network host clickhouse/integration-tests-runner:c909f2e80ec78b3a6532 \
+            /bin/bash -c "
+              CLICKHOUSE_TESTS_SERVER_BIN_PATH=$PWD/build/programs/clickhouse \
+              pytest $PWD/tests/integration/test_amateur_merge_tree/ -v --timeout=600
+            "

tests/integration/test_amateur_merge_tree/__init__.py

@@ -0,0 +1,19 @@
+---
+enforce_keeper_component_tracking: false
+
+storage_configuration:
+  disks:
+    s3:
+      type: object_storage
+      object_storage_type: s3
+      metadata_type: amateur
+      endpoint: http://minio1:9001/root/data/
+      keeper_prefix: /clickhouse/disks/s3
+      access_key_id: minio
+      secret_access_key: ClickHouse_Minio_P@ssw0rd
+  policies:
+    s3:
+      volumes:
+      - main:
+          disks:
+          - s3

tests/integration/test_amateur_merge_tree/configs/config.yaml

@@ -0,0 +1,87 @@
+#!/usr/bin/env python3
+
+import time
+
+import pytest
+
+from helpers.cluster import ClickHouseCluster
+from helpers.test_tools import TSV
+
+cluster = ClickHouseCluster(__file__)
+
+
+def wait_table_sync(zk_client, zk_table_path):
+    _, parts_stat = zk_client.get(f"{zk_table_path}/parts")
+    nodes = zk_client.get_children(f"{zk_table_path}/replicas")
+
+    for _ in range(60):
+        nodes_in_sync = 0
+        for node in nodes:
+            node_epoch, _ = zk_client.get(f"{zk_table_path}/replicas/{node}/epoch")
+            if int(node_epoch) >= parts_stat.pzxid:
+                nodes_in_sync += 1
+
+        if nodes_in_sync == len(nodes):
+            return
+
+        time.sleep(0.5)
+
+
+@pytest.fixture(scope="module")
+def started_cluster():
+    try:
+        cluster.add_instance(
+            "node1",
+            main_configs=["configs/config.yaml"],
+            with_minio=True,
+            with_zookeeper=True,
+            stay_alive=True,
+            use_keeper=False,
+            cpu_limit=3,
+        )
+        cluster.add_instance(
+            "node2",
+            main_configs=["configs/config.yaml"],
+            with_minio=True,
+            with_zookeeper=True,
+            stay_alive=True,
+            use_keeper=False,
+            cpu_limit=3
+        )
+        cluster.start()
+        yield cluster
+    finally:
+        cluster.shutdown()
+
+
+def test_insert_select(started_cluster):
+    """
+    Basic scenario with inserts and replica synchronization
+    """
+
+    for node_name, node in cluster.instances.items():
+        data = node.query(f"""
+            CREATE TABLE test_basic (key UInt32, val String)
+            ENGINE = AmateurMergeTree('/clickhouse/tables/default/test_basic', '{node_name}')
+            ORDER BY key
+            SETTINGS storage_policy = 's3'
+        """)
+
+    cluster.instances["node1"].query("""
+        INSERT INTO test_basic SELECT number, 'val-' || number FROM numbers(10)
+    """)
+
+    cluster.instances["node2"].query("""
+        INSERT INTO test_basic SELECT number + 10, 'val-' || (number + 10) FROM numbers(10)
+    """)
+
+    zk_client = cluster.get_kazoo_client("zoo1")
+    wait_table_sync(zk_client, "/clickhouse/tables/default/test_basic")
+
+    expected = [[0, 'val-0', 19, 'val-19']]
+    for node_name, node in cluster.instances.items():
+        actual = node.query("""
+            SELECT min(key), argMin(val, key), max(key), argMax(val, key) FROM test_basic
+        """)
+
+        assert TSV(actual) == TSV(expected), f"Received an unexpected result from node {node_name}"