Skip to content

Commit ccebe67

Browse files
committed
Fix deferred freeClient clobbering replication state after replicaof
Since PR #3324, freeClient() on a primary client with pending IO is deferred via freeClientAsync. The deferred free eventually chains through replicationCachePrimary() -> replicationHandlePrimaryDisconnection(), which unconditionally set repl_state = REPL_STATE_CONNECT. This causes two bugs: 1. REPLICAOF NO ONE: primary_host is NULL when the deferred free runs, so replicationCron calls connectWithPrimary(NULL) -> SIGSEGV in connTLSConnect (inet_pton with NULL addr). 2. REPLICAOF newhost newport: the deferred free clobbers the already- progressed repl_state (CONNECTING) back to CONNECT, causing replicationCron to call connectWithPrimary() again, which overwrites server.repl_transfer_s without closing the previous connection (FD leak). Fix by making replicationHandlePrimaryDisconnection() only transition to REPL_STATE_CONNECT when repl_state is still REPL_STATE_CONNECTED (meaning this is a genuine disconnect, not a stale deferred free). If repl_state has already moved on, the deferred free is stale and should not mutate the state machine. Additionally: - Add NULL check for addr in connTLSConnect() as defense in depth. - Add 10s timeout to the WAITAOF test to prevent indefinite hanging. - Add dedicated tests for the repoint scenario. Signed-off-by: Yaron Sananes <yaron.sananes@gmail.com>
1 parent fdf13ca commit ccebe67

7 files changed

Lines changed: 80 additions & 4 deletions

File tree

src/debug.c

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1066,6 +1066,9 @@ void debugCommand(client *c) {
10661066
} else if (!strcasecmp(objectGetVal(c->argv[1]), "client-enforce-reply-list") && c->argc == 3) {
10671067
server.debug_client_enforce_reply_list = atoi(objectGetVal(c->argv[2]));
10681068
addReply(c, shared.ok);
1069+
} else if (!strcasecmp(objectGetVal(c->argv[1]), "force-free-primary-async") && c->argc == 3) {
1070+
server.debug_force_free_primary_async = atoi(objectGetVal(c->argv[2]));
1071+
addReply(c, shared.ok);
10691072
} else if (!handleDebugClusterCommand(c)) {
10701073
addReplySubcommandSyntaxError(c);
10711074
return;

src/networking.c

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2102,6 +2102,14 @@ int freeClient(client *c) {
21022102
return 0;
21032103
}
21042104

2105+
/* Debug: force async free for the primary client to deterministically
2106+
* reproduce the deferred-free replication state clobber race. */
2107+
if (server.debug_force_free_primary_async && c->flag.primary) {
2108+
server.debug_force_free_primary_async = 0;
2109+
freeClientAsync(c);
2110+
return 0;
2111+
}
2112+
21052113
/* For connected clients, call the disconnection event of modules hooks. */
21062114
if (c->conn) {
21072115
moduleFireServerEvent(VALKEYMODULE_EVENT_CLIENT_CHANGE, VALKEYMODULE_SUBEVENT_CLIENT_CHANGE_DISCONNECTED, c);

src/replication.c

Lines changed: 19 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -4560,15 +4560,31 @@ void replicationHandlePrimaryDisconnection(void) {
45604560
moduleFireServerEvent(VALKEYMODULE_EVENT_PRIMARY_LINK_CHANGE, VALKEYMODULE_SUBEVENT_PRIMARY_LINK_DOWN, NULL);
45614561

45624562
server.primary = NULL;
4563-
server.repl_state = REPL_STATE_CONNECT;
4564-
server.repl_down_since = server.unixtime;
4563+
4564+
/* freeClient(primary) can be deferred via freeClientAsync when the client
4565+
* has pending IO. By the time we run in that deferred context,
4566+
* replicationUnsetPrimary()/replicationSetPrimary() may have already
4567+
* finalized replication state. Only transition to REPL_STATE_CONNECT if
4568+
* we were genuinely connected (REPL_STATE_CONNECTED) and primary_host is
4569+
* still set. Otherwise this is a stale deferred free and we must not
4570+
* clobber the current state. */
4571+
if (server.repl_state == REPL_STATE_CONNECTED && server.primary_host) {
4572+
server.repl_state = REPL_STATE_CONNECT;
4573+
server.repl_down_since = server.unixtime;
4574+
} else if (server.repl_state == REPL_STATE_CONNECTED) {
4575+
/* primary_host is NULL: deliberate unset in progress. */
4576+
server.repl_state = REPL_STATE_NONE;
4577+
}
4578+
/* Any other repl_state means the state machine already moved on
4579+
* (e.g. REPL_STATE_CONNECT, CONNECTING, NONE) — leave it untouched. */
4580+
45654581
/* We lost connection with our primary, don't disconnect replicas yet,
45664582
* maybe we'll be able to PSYNC with our primary later. We'll disconnect
45674583
* the replicas only if we'll have to do a full resync with our primary. */
45684584

45694585
/* Try to re-connect immediately rather than wait for replicationCron
45704586
* waiting 1 second may risk backlog being recycled. */
4571-
if (server.primary_host) {
4587+
if (server.repl_state == REPL_STATE_CONNECT && server.primary_host) {
45724588
serverLog(LL_NOTICE, "Reconnecting to PRIMARY %s:%d", server.primary_host, server.primary_port);
45734589
connectWithPrimary();
45744590
}

src/server.c

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2967,6 +2967,7 @@ void initServer(void) {
29672967
server.reply_buffer_resizing_enabled = 1;
29682968
server.client_mem_usage_buckets = NULL;
29692969
server.debug_client_enforce_reply_list = 0;
2970+
server.debug_force_free_primary_async = 0;
29702971
resetReplicationBuffer();
29712972

29722973
/* Make sure the locale is set on startup based on the config file. */

src/server.h

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1853,6 +1853,7 @@ struct valkeyServer {
18531853
int enable_module_cmd; /* Enable MODULE commands, see PROTECTED_ACTION_ALLOWED_* */
18541854
int enable_debug_assert; /* Enable debug asserts */
18551855
int debug_client_enforce_reply_list; /* Force client to always use the reply list */
1856+
int debug_force_free_primary_async; /* Force freeClient on primary to use async path */
18561857
/* Reply construction copy avoidance */
18571858
int min_io_threads_copy_avoid; /* Minimum number of IO threads for copy avoidance in reply construction */
18581859
int min_string_size_copy_avoid_threaded; /* Minimum bulk string size for copy avoidance in reply construction when IO threads enabled */

src/tls.c

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1603,6 +1603,7 @@ static int connTLSConnect(connection *conn_,
16031603
unsigned char addr_buf[sizeof(struct in6_addr)];
16041604

16051605
if (conn->c.state != CONN_STATE_NONE) return C_ERR;
1606+
if (addr == NULL) return C_ERR;
16061607
ERR_clear_error();
16071608

16081609
/* Check whether addr is an IP address, if not, use the value for Server Name Indication */

tests/unit/wait.tcl

Lines changed: 47 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -343,7 +343,7 @@ tags {"wait aof network external:skip"} {
343343
set rd [valkey_deferring_client -1]
344344
$rd incr foo
345345
$rd read
346-
$rd waitaof 0 1 0
346+
$rd waitaof 0 1 10000
347347
wait_for_blocked_client -1
348348
$replica replicaof $master_host $master_port
349349
assert_equal [$rd read] {1 1}
@@ -532,3 +532,49 @@ start_server {} {
532532
}
533533
}
534534
}
535+
536+
start_server {tags {"wait network external:skip"}} {
537+
start_server {} {
538+
start_server {} {
539+
set master1 [srv -2 client]
540+
set master1_host [srv -2 host]
541+
set master1_port [srv -2 port]
542+
543+
set master2 [srv -1 client]
544+
set master2_host [srv -1 host]
545+
set master2_port [srv -1 port]
546+
547+
set replica [srv 0 client]
548+
549+
test {Repoint replica with deferred freeClient does not double-connect} {
550+
$replica replicaof $master1_host $master1_port
551+
wait_for_condition 50 100 {
552+
[s 0 master_link_status] eq {up}
553+
} else {
554+
fail "Replication to master1 not established"
555+
}
556+
557+
$replica debug force-free-primary-async 1
558+
559+
set log_lines [count_log_lines 0]
560+
$replica replicaof $master2_host $master2_port
561+
wait_for_condition 50 200 {
562+
[s 0 master_link_status] eq {up}
563+
} else {
564+
fail "Replication to master2 not established after repoint"
565+
}
566+
after 2000
567+
568+
set logfile [srv 0 stdout]
569+
set lines [split [exec tail -n +[expr {$log_lines + 1}] < $logfile] "\n"]
570+
set sync_count 0
571+
foreach line $lines {
572+
if {[string match "*Connecting to PRIMARY $master2_host:$master2_port*" $line]} {
573+
incr sync_count
574+
}
575+
}
576+
assert_equal 1 $sync_count
577+
}
578+
}
579+
}
580+
}

0 commit comments

Comments
 (0)