Skip to content

Commit fefeaf0

Browse files
authored
branch-3.1: [fix](sc) Skip empty rowset version hole filling for versions <= alter_version #56209 (#56212)
cherry pick from #56209
1 parent 7d3e601 commit fefeaf0

6 files changed

Lines changed: 148 additions & 3 deletions

File tree

be/src/cloud/cloud_delta_writer.cpp

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,9 @@
2626

2727
namespace doris {
2828

29+
bvar::Adder<int64_t> g_cloud_commit_rowset_count("cloud_commit_rowset_count");
30+
bvar::Adder<int64_t> g_cloud_commit_empty_rowset_count("cloud_commit_empty_rowset_count");
31+
2932
CloudDeltaWriter::CloudDeltaWriter(CloudStorageEngine& engine, const WriteRequest& req,
3033
RuntimeProfile* profile, const UniqueId& load_id)
3134
: BaseDeltaWriter(req, profile, load_id), _engine(engine) {
@@ -108,10 +111,12 @@ const RowsetMetaSharedPtr& CloudDeltaWriter::rowset_meta() {
108111
}
109112

110113
Status CloudDeltaWriter::commit_rowset() {
114+
g_cloud_commit_rowset_count << 1;
111115
std::lock_guard<bthread::Mutex> lock(_mtx);
112116

113117
// Handle empty rowset (no data written)
114118
if (!_is_init) {
119+
g_cloud_commit_empty_rowset_count << 1;
115120
return _commit_empty_rowset();
116121
}
117122

be/src/cloud/cloud_meta_mgr.cpp

Lines changed: 35 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1706,28 +1706,61 @@ Status CloudMetaMgr::fill_version_holes(CloudTablet* tablet, int64_t max_version
17061706
return a.first < b.first;
17071707
});
17081708

1709+
// During schema change, get_tablet operations on new tablets trigger sync_tablet_rowsets which calls
1710+
// fill_version_holes. For schema change tablets (TABLET_NOTREADY state), we selectively skip hole
1711+
// filling for versions <= alter_version to prevent:
1712+
// 1. Abnormal compaction score calculations for schema change tablets
1713+
// 2. Unexpected -235 errors during load operations
1714+
// This allows schema change to proceed normally while still permitting hole filling for versions
1715+
// beyond the alter_version threshold.
1716+
bool is_schema_change_tablet = tablet->tablet_state() == TABLET_NOTREADY;
1717+
if (is_schema_change_tablet && tablet->alter_version() <= 1) {
1718+
LOG(INFO) << "Skip version hole filling for new schema change tablet "
1719+
<< tablet->tablet_id() << " with alter_version " << tablet->alter_version();
1720+
return Status::OK();
1721+
}
1722+
17091723
int64_t last_version = -1;
17101724
for (const Version& version : existing_versions) {
1725+
VLOG_NOTICE << "Existing version for tablet " << tablet->tablet_id() << ": ["
1726+
<< version.first << ", " << version.second << "]";
17111727
// missing versions are those that are not in the existing_versions
17121728
if (version.first > last_version + 1) {
17131729
// there is a hole between versions
17141730
auto prev_non_hole_rowset = tablet->get_rowset_by_version(version);
17151731
for (int64_t ver = last_version + 1; ver < version.first; ++ver) {
1732+
// Skip hole filling for versions <= alter_version during schema change
1733+
if (is_schema_change_tablet && ver <= tablet->alter_version()) {
1734+
continue;
1735+
}
17161736
RowsetSharedPtr hole_rowset;
17171737
RETURN_IF_ERROR(create_empty_rowset_for_hole(
17181738
tablet, ver, prev_non_hole_rowset->rowset_meta(), &hole_rowset));
17191739
hole_rowsets.push_back(hole_rowset);
17201740
}
17211741
LOG(INFO) << "Created empty rowset for version hole, from " << last_version + 1
1722-
<< " to " << version.first - 1 << " for tablet " << tablet->tablet_id();
1742+
<< " to " << version.first - 1 << " for tablet " << tablet->tablet_id()
1743+
<< (is_schema_change_tablet
1744+
? (", schema change tablet skipped filling versions <= " +
1745+
std::to_string(tablet->alter_version()))
1746+
: "");
17231747
}
17241748
last_version = version.second;
17251749
}
17261750

17271751
if (last_version + 1 <= max_version) {
17281752
LOG(INFO) << "Created empty rowset for version hole, from " << last_version + 1 << " to "
1729-
<< max_version << " for tablet " << tablet->tablet_id();
1753+
<< max_version << " for tablet " << tablet->tablet_id()
1754+
<< (is_schema_change_tablet
1755+
? (", schema change tablet skipped filling versions <= " +
1756+
std::to_string(tablet->alter_version()))
1757+
: "");
1758+
// there is a hole after the last existing version
17301759
for (; last_version + 1 <= max_version; ++last_version) {
1760+
// Skip hole filling for versions <= alter_version during schema change
1761+
if (is_schema_change_tablet && last_version + 1 <= tablet->alter_version()) {
1762+
continue;
1763+
}
17311764
RowsetSharedPtr hole_rowset;
17321765
auto prev_non_hole_rowset = tablet->get_rowset_by_version(existing_versions.back());
17331766
RETURN_IF_ERROR(create_empty_rowset_for_hole(

be/src/cloud/cloud_schema_change_job.cpp

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919

2020
#include <gen_cpp/cloud.pb.h>
2121

22+
#include <algorithm>
2223
#include <chrono>
2324
#include <memory>
2425
#include <random>
@@ -459,12 +460,22 @@ Status CloudSchemaChangeJob::_process_delete_bitmap(int64_t alter_version,
459460
RETURN_IF_ERROR(_cloud_storage_engine.register_compaction_stop_token(_new_tablet, initiator));
460461
TabletMetaSharedPtr tmp_meta = std::make_shared<TabletMeta>(*(_new_tablet->tablet_meta()));
461462
tmp_meta->delete_bitmap()->delete_bitmap.clear();
462-
tmp_meta->clear_rowsets();
463+
// Keep only version [0-1] rowset, other rowsets will be added in _output_rowsets
464+
auto& rs_metas = tmp_meta->all_mutable_rs_metas();
465+
rs_metas.erase(std::remove_if(rs_metas.begin(), rs_metas.end(),
466+
[](const RowsetMetaSharedPtr& rs_meta) {
467+
return !(rs_meta->version().first == 0 &&
468+
rs_meta->version().second == 1);
469+
}),
470+
rs_metas.end());
471+
463472
std::shared_ptr<CloudTablet> tmp_tablet =
464473
std::make_shared<CloudTablet>(_cloud_storage_engine, tmp_meta);
465474
{
466475
std::unique_lock wlock(tmp_tablet->get_header_lock());
467476
tmp_tablet->add_rowsets(_output_rowsets, true, wlock);
477+
// Set alter version to let the tmp_tablet can fill hole rowset greater than alter_version
478+
tmp_tablet->set_alter_version(alter_version);
468479
}
469480

470481
// step 1, process incremental rowset without delete bitmap update lock
Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
-- This file is automatically generated. You should know what you did if you want to edit this
2+
-- !sql --
3+
7140 240
4+

regression-test/suites/schema_change_p0/test_schema_change_mow_with_empty_rowset.groovy

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -62,6 +62,7 @@ suite("test_schema_change_mow_with_empty_rowset", "p0") {
6262
for (int i = 0; i < 20; i++) {
6363
sql """ insert into ${tableName} values (100, 2, 3, 4, 5, 6.6, 1.7, 8.8,
6464
'a', 'b', 'c', '2021-10-30', '2021-10-30 00:00:00') """
65+
sleep(20)
6566
}
6667

6768
Awaitility.await().atMost(30, TimeUnit.SECONDS).pollDelay(10, TimeUnit.MILLISECONDS).pollInterval(10, TimeUnit.MILLISECONDS).until(
Lines changed: 91 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,91 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
import java.util.concurrent.TimeUnit
19+
import org.awaitility.Awaitility
20+
21+
suite("test_schema_change_with_empty_rowset", "p0,nonConcurrent") {
22+
def custoBeConfig = [
23+
max_tablet_version_num : 100
24+
]
25+
26+
setBeConfigTemporary(custoBeConfig) {
27+
def tableName = "test_sc_with_empty_rowset"
28+
29+
def getJobState = { tbl ->
30+
def jobStateResult = sql """ SHOW ALTER TABLE COLUMN WHERE IndexName='${tbl}' ORDER BY createtime DESC LIMIT 1 """
31+
return jobStateResult[0][9]
32+
}
33+
34+
sql """ DROP TABLE IF EXISTS ${tableName} """
35+
36+
sql """
37+
CREATE TABLE IF NOT EXISTS ${tableName} (
38+
`k1` int(11) NULL,
39+
`k2` tinyint(4) NULL,
40+
`k3` smallint(6) NULL,
41+
`k4` int(30) NULL,
42+
`k5` largeint(40) NULL,
43+
`k6` float NULL,
44+
`k7` double NULL,
45+
`k8` decimal(9, 0) NULL,
46+
`k9` char(10) NULL,
47+
`k10` varchar(1024) NULL,
48+
`k11` text NULL,
49+
`k12` date NULL,
50+
`k13` datetime NULL
51+
) ENGINE=OLAP
52+
UNIQUE KEY(k1, k2, k3)
53+
DISTRIBUTED BY HASH(`k1`) BUCKETS 2
54+
PROPERTIES (
55+
"replication_allocation" = "tag.location.default: 1",
56+
"enable_unique_key_merge_on_write" = "true"
57+
);
58+
"""
59+
60+
for (int i = 0; i < 100; i++) {
61+
sql """ insert into ${tableName} values ($i, 2, 3, 4, 5, 6.6, 1.7, 8.8,
62+
'a', 'b', 'c', '2021-10-30', '2021-10-30 00:00:00') """
63+
}
64+
65+
66+
// trigger compactions for all tablets in ${tableName}
67+
trigger_and_wait_compaction(tableName, "cumulative")
68+
69+
sql """ alter table ${tableName} modify column k4 string NULL"""
70+
71+
for (int i = 100; i < 120; i++) {
72+
sql """ insert into ${tableName} values ($i, 2, 3, 4, 5, 6.6, 1.7, 8.8,
73+
'a', 'b', 'c', '2021-10-30', '2021-10-30 00:00:00') """
74+
sleep(20)
75+
}
76+
77+
Awaitility.await().atMost(30, TimeUnit.SECONDS).pollDelay(10, TimeUnit.MILLISECONDS).pollInterval(10, TimeUnit.MILLISECONDS).until(
78+
{
79+
String res = getJobState(tableName)
80+
if (res == "FINISHED" || res == "CANCELLED") {
81+
assertEquals("FINISHED", res)
82+
return true
83+
}
84+
return false
85+
}
86+
)
87+
88+
qt_sql """ select sum(k1), sum(k2) from ${tableName} """
89+
}
90+
}
91+

0 commit comments

Comments
 (0)