Skip to content

Commit 24c9b5a

Browse files
perf(strcasecmp): fused SWAR case-fold compare for strcasecmp/strncasecmp (bd-2g7oyh.362)
strcasecmp did THREE passes (two full scan_c_string length scans + a separate core SIMD compare), computing both lengths eagerly even on an early mismatch; strncasecmp was a byte-at-a-time tolower loop. Replace both with one fused single-pass scan_strcasecmp that compares case-folded 8-byte words and early- exits at the first diff or NUL. Add swar_ascii_lower: branchless SWAR ASCII lowercase (folds only 'A'..='Z', non-ASCII untouched = C/POSIX-locale tolower on all 8 lanes). The per-byte range test is made borrow-safe by forcing each byte's high bit (w | HIGHS) so a within-byte borrow is absorbed by that guard bit instead of leaking into the next lane; an ascii mask (!w & HIGHS) excludes bytes >= 0x80. Equal+NUL-free folded windows advance 8; other windows resolve byte-wise with to_ascii_lowercase (byte-identical to the scalar loop). Dual-pointer wide reads are page-cross guarded (wide_read_within_page) like strcmp. strcasecmp keeps the exact original clamped-slice semantics on the rare membrane-repair path. Measured (rch, median ns/op, equal-mod-case full scan isolated): len old(ns) new(ns) glibc(ns) self vs-glibc 16 13.8 7.1 21.0 1.93x 2.95x 64 57.7 17.3 31.1 3.34x 1.80x 256 206.4 57.8 71.5 3.57x 1.24x 4096 3228.5 879.0 895.2 3.67x 1.02x 65536 53117.4 13791.6 13836.8 3.85x 1.00x Self-speedup 3.3-3.9x for n>=64 (>=2.0); matches-or-beats glibc every size. The old byte loop shown is strncasecmp's; strcasecmp's old 3-pass path was slower still, so its win is larger. Parity: conformance_diff_strcasecmp.rs proves (1) swar_ascii_lower == scalar to_ascii_lowercase for ALL 256 byte values in ALL 8 lanes (carry-safety + only A-Z fold); (2) strcasecmp/strncasecmp sign-match host glibc across 100k+ pairs (equal-mod-case/case-swap/single-byte-diff/shared+early NUL/high-bit x alignment 0..8 x len 0..68 x n straddling len); (3) guard-page over-read test. Ninth ABI string-vein win. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
1 parent 5c5b4a6 commit 24c9b5a

3 files changed

Lines changed: 326 additions & 26 deletions

File tree

crates/frankenlibc-abi/src/string_abi.rs

Lines changed: 116 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -621,6 +621,23 @@ pub unsafe fn bench_scan_c_string_last_byte(
621621
unsafe { scan_c_string_last_byte(ptr, target, bound) }
622622
}
623623

624+
/// Test hook: per-lane SWAR ASCII lowercase, for exhaustive parity vs
625+
/// `to_ascii_lowercase`. Not part of the public ABI.
626+
#[doc(hidden)]
627+
pub fn test_swar_ascii_lower(w: u64) -> u64 {
628+
swar_ascii_lower(w)
629+
}
630+
631+
/// Benchmark/test hook for the fused SWAR [`scan_strcasecmp`] (behind
632+
/// strcasecmp/strncasecmp). Not part of the public ABI.
633+
///
634+
/// # Safety
635+
/// `s1`/`s2` must be NUL-terminated, or valid for `bound` bytes.
636+
#[doc(hidden)]
637+
pub unsafe fn bench_scan_strcasecmp(s1: *const c_char, s2: *const c_char, bound: usize) -> c_int {
638+
unsafe { scan_strcasecmp(s1, s2, bound).0 }
639+
}
640+
624641
#[inline]
625642
unsafe fn copy_unaligned_16(dst: *mut u8, src: *const u8) {
626643
// SAFETY: caller guarantees 16 readable/writable bytes.
@@ -1242,6 +1259,86 @@ unsafe fn scan_strcmp(s1: *const c_char, s2: *const c_char, bound: usize) -> (us
12421259
}
12431260
}
12441261

1262+
/// Branchless SWAR ASCII lowercase: folds bytes in `'A'..='Z'` to `'a'..='z'`
1263+
/// and leaves every other byte (incl. non-ASCII `>= 0x80`) untouched — exactly C
1264+
/// `tolower` in the POSIX/C locale, applied to all 8 lanes at once.
1265+
///
1266+
/// Per-byte range test `0x41 <= b <= 0x5A`, made borrow-safe by forcing each
1267+
/// byte's high bit (`w | HIGHS`) so a within-byte borrow is absorbed by that
1268+
/// guard bit instead of leaking into the next lane. `ge_a`/`ge_5b` read the
1269+
/// surviving guard bit as `(b & 0x7F) >= 0x41` / `>= 0x5B`; `ascii` excludes
1270+
/// bytes `>= 0x80`. The resulting `0x80` flag is shifted to the `0x20` case bit.
1271+
#[inline(always)]
1272+
fn swar_ascii_lower(w: u64) -> u64 {
1273+
const ONES: u64 = 0x0101_0101_0101_0101;
1274+
const HIGHS: u64 = 0x8080_8080_8080_8080;
1275+
let guarded = w | HIGHS;
1276+
let ge_a = guarded.wrapping_sub(ONES.wrapping_mul(0x41)) & HIGHS; // (b&0x7F) >= 'A'
1277+
let ge_5b = guarded.wrapping_sub(ONES.wrapping_mul(0x5B)) & HIGHS; // (b&0x7F) >= '['
1278+
let ascii = !w & HIGHS; // b < 0x80
1279+
let is_upper = ge_a & !ge_5b & ascii;
1280+
w | (is_upper >> 2)
1281+
}
1282+
1283+
/// Fused single-pass SWAR case-insensitive compare of two C strings within
1284+
/// `bound`. Returns `(result, span)`: `result` is the signed difference of the
1285+
/// lowercased bytes at the first position that differs (0 if equal up to a shared
1286+
/// NUL or to `bound`); `span` is the compared extent (for cost accounting).
1287+
///
1288+
/// Equal-and-NUL-free 8-byte windows (after folding both via [`swar_ascii_lower`])
1289+
/// advance 8; any other window is resolved byte-wise with `to_ascii_lowercase`
1290+
/// (byte-identical to the scalar loop). The same page-cross guard as
1291+
/// [`scan_strcmp`] keeps the dual-pointer wide reads from faulting past a NUL.
1292+
unsafe fn scan_strcasecmp(s1: *const c_char, s2: *const c_char, bound: usize) -> (c_int, usize) {
1293+
let p1 = s1.cast::<u8>();
1294+
let p2 = s2.cast::<u8>();
1295+
let mut i = 0usize;
1296+
loop {
1297+
if i + 8 <= bound
1298+
&& wide_read_within_page(p1 as usize + i)
1299+
&& wide_read_within_page(p2 as usize + i)
1300+
{
1301+
// SAFETY: both 8-byte reads stay within their mapped pages and bound.
1302+
let wa = unsafe { core::ptr::read_unaligned(p1.add(i).cast::<u64>()) };
1303+
let wb = unsafe { core::ptr::read_unaligned(p2.add(i).cast::<u64>()) };
1304+
if swar_ascii_lower(wa) == swar_ascii_lower(wb) && !swar_word_has_zero(wa) {
1305+
i += 8;
1306+
continue;
1307+
}
1308+
for j in 0..8 {
1309+
// SAFETY: i+j < bound; within the just-read in-page window.
1310+
let a = unsafe { *p1.add(i + j) };
1311+
let b = unsafe { *p2.add(i + j) };
1312+
let la = a.to_ascii_lowercase();
1313+
let lb = b.to_ascii_lowercase();
1314+
if la != lb {
1315+
return ((la as c_int) - (lb as c_int), i + j + 1);
1316+
}
1317+
if a == 0 {
1318+
return (0, i + j + 1);
1319+
}
1320+
}
1321+
i += 8; // defensive: a flagged window always returns above.
1322+
continue;
1323+
}
1324+
if i >= bound {
1325+
return (0, bound);
1326+
}
1327+
// SAFETY: i < bound.
1328+
let a = unsafe { *p1.add(i) };
1329+
let b = unsafe { *p2.add(i) };
1330+
let la = a.to_ascii_lowercase();
1331+
let lb = b.to_ascii_lowercase();
1332+
if la != lb {
1333+
return ((la as c_int) - (lb as c_int), i + 1);
1334+
}
1335+
if a == 0 {
1336+
return (0, i + 1);
1337+
}
1338+
i += 1;
1339+
}
1340+
}
1341+
12451342
unsafe fn read_c_string_bytes(ptr: *const c_char) -> Option<Vec<u8>> {
12461343
if ptr.is_null() {
12471344
return None;
@@ -3610,14 +3707,22 @@ pub unsafe extern "C" fn strcasecmp(s1: *const c_char, s2: *const c_char) -> c_i
36103707

36113708
// SAFETY: bounded scan within known limits.
36123709
let (result, span) = unsafe {
3613-
let (s1_len, s1_term) = scan_c_string(s1, lhs_bound);
3614-
let (s2_len, s2_term) = scan_c_string(s2, rhs_bound);
3615-
let s1_slice_len = if s1_term { s1_len + 1 } else { s1_len };
3616-
let s2_slice_len = if s2_term { s2_len + 1 } else { s2_len };
3617-
let s1_slice = std::slice::from_raw_parts(s1.cast::<u8>(), s1_slice_len);
3618-
let s2_slice = std::slice::from_raw_parts(s2.cast::<u8>(), s2_slice_len);
3619-
let r = frankenlibc_core::string::str::strcasecmp(s1_slice, s2_slice);
3620-
(r, s1_len.max(s2_len))
3710+
if lhs_bound.is_none() && rhs_bound.is_none() {
3711+
// Common path: one fused SWAR case-compare with early exit, instead of
3712+
// two full length scans plus a separate compare pass.
3713+
scan_strcasecmp(s1, s2, usize::MAX)
3714+
} else {
3715+
// Repair path: preserve the exact clamped-slice semantics (out-of-bound
3716+
// bytes treated as NUL by the core comparator).
3717+
let (s1_len, s1_term) = scan_c_string(s1, lhs_bound);
3718+
let (s2_len, s2_term) = scan_c_string(s2, rhs_bound);
3719+
let s1_slice_len = if s1_term { s1_len + 1 } else { s1_len };
3720+
let s2_slice_len = if s2_term { s2_len + 1 } else { s2_len };
3721+
let s1_slice = std::slice::from_raw_parts(s1.cast::<u8>(), s1_slice_len);
3722+
let s2_slice = std::slice::from_raw_parts(s2.cast::<u8>(), s2_slice_len);
3723+
let r = frankenlibc_core::string::str::strcasecmp(s1_slice, s2_slice);
3724+
(r, s1_len.max(s2_len))
3725+
}
36213726
};
36223727

36233728
record_string_stage_outcome(
@@ -3700,23 +3805,9 @@ pub unsafe extern "C" fn strncasecmp(s1: *const c_char, s2: *const c_char, n: us
37003805
let adverse = repair && cmp_limit < n;
37013806

37023807
// SAFETY: bounded compare within cmp_limit.
3703-
let result = unsafe {
3704-
let mut i = 0usize;
3705-
loop {
3706-
if i >= cmp_limit {
3707-
break 0;
3708-
}
3709-
let a = (*s1.add(i) as u8).to_ascii_lowercase();
3710-
let b = (*s2.add(i) as u8).to_ascii_lowercase();
3711-
if a != b {
3712-
break (a as c_int) - (b as c_int);
3713-
}
3714-
if a == 0 {
3715-
break 0;
3716-
}
3717-
i += 1;
3718-
}
3719-
};
3808+
// Fused SWAR case-compare (shared scan_strcasecmp), byte-identical to the old
3809+
// scalar tolower loop; bounded by cmp_limit and page-cross guarded.
3810+
let result = unsafe { scan_strcasecmp(s1, s2, cmp_limit).0 };
37203811

37213812
if adverse {
37223813
record_truncation(n, cmp_limit);
Lines changed: 156 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,156 @@
1+
//! Gate for the public `strcasecmp`/`strncasecmp` ABI after fusing their scans
2+
//! into a single SWAR case-fold compare.
3+
//!
4+
//! 1. The branchless SWAR ASCII lowercase must equal `u8::to_ascii_lowercase` for
5+
//! EVERY byte value 0..=255 in EVERY lane position (carry-safety + C-locale:
6+
//! only `A`-`Z` fold, non-ASCII untouched).
7+
//! 2. fl must agree (in sign) with host glibc `strcasecmp`/`strncasecmp` across
8+
//! many pairs (equal mod case, case-only diff, single-byte diff, shared/early
9+
//! NUL, high-bit bytes, alignments, lengths, strncasecmp n straddling len).
10+
//! 3. The wide reads must not fault past a NUL flush against an unmapped page.
11+
#![cfg(target_os = "linux")]
12+
#![allow(unsafe_code)]
13+
14+
use frankenlibc_abi::string_abi::{
15+
strcasecmp as fl_strcasecmp, strncasecmp as fl_strncasecmp, test_swar_ascii_lower,
16+
};
17+
use std::os::raw::c_char;
18+
19+
fn sign(x: i32) -> i32 {
20+
x.signum()
21+
}
22+
23+
#[test]
24+
fn swar_ascii_lower_matches_scalar_for_all_bytes_and_lanes() {
25+
for lane in 0..8 {
26+
for b in 0u16..=255 {
27+
let b = b as u8;
28+
let w = (b as u64) << (lane * 8);
29+
let folded = test_swar_ascii_lower(w);
30+
let got = ((folded >> (lane * 8)) & 0xFF) as u8;
31+
assert_eq!(
32+
got,
33+
b.to_ascii_lowercase(),
34+
"swar lower byte={b:#x} lane={lane}: got {got:#x}"
35+
);
36+
// Other lanes must be untouched (they were 0).
37+
let cleared = folded & !(0xFFu64 << (lane * 8));
38+
assert_eq!(cleared, 0, "swar lower leaked into other lanes for byte={b:#x} lane={lane}");
39+
}
40+
}
41+
// A full mixed word.
42+
let w = u64::from_ne_bytes(*b"AbZ@[a1\xff");
43+
let got = test_swar_ascii_lower(w).to_ne_bytes();
44+
let want: Vec<u8> = b"AbZ@[a1\xff".iter().map(|c| c.to_ascii_lowercase()).collect();
45+
assert_eq!(&got[..], &want[..], "swar lower mixed word");
46+
}
47+
48+
#[test]
49+
fn strcasecmp_strncasecmp_match_glibc() {
50+
let mut checked = 0u64;
51+
let alphabet = b"aB\x80Cd\xffeF";
52+
for align1 in 0usize..8 {
53+
for align2 in 0usize..8 {
54+
for len in 0usize..68 {
55+
let mut base: Vec<u8> = (0..len).map(|k| alphabet[k % alphabet.len()]).collect();
56+
base.push(0);
57+
58+
let mut variants: Vec<Vec<u8>> = vec![base.clone()];
59+
// Case-swap each letter; single-byte mutations.
60+
let mut swapped = base.clone();
61+
for b in swapped.iter_mut() {
62+
if b.is_ascii_alphabetic() {
63+
*b ^= 0x20;
64+
}
65+
}
66+
variants.push(swapped);
67+
for pos in 0..base.len() {
68+
for &nb in &[b'a', b'Z', 0x80u8, 0xFFu8, 0u8] {
69+
if base[pos] != nb {
70+
let mut v = base.clone();
71+
v[pos] = nb;
72+
variants.push(v);
73+
}
74+
}
75+
}
76+
77+
let mut b1 = vec![0u64; (align1 + base.len()) / 8 + 2];
78+
let p1base = b1.as_mut_ptr().cast::<u8>();
79+
unsafe {
80+
for (k, &b) in base.iter().enumerate() {
81+
*p1base.add(align1 + k) = b;
82+
}
83+
}
84+
let p1 = unsafe { p1base.add(align1) } as *const c_char;
85+
86+
for v in &variants {
87+
let mut b2 = vec![0u64; (align2 + v.len()) / 8 + 2];
88+
let p2base = b2.as_mut_ptr().cast::<u8>();
89+
unsafe {
90+
for (k, &b) in v.iter().enumerate() {
91+
*p2base.add(align2 + k) = b;
92+
}
93+
}
94+
let p2 = unsafe { p2base.add(align2) } as *const c_char;
95+
96+
let fl = unsafe { fl_strcasecmp(p1, p2) };
97+
let gl = unsafe { libc::strcasecmp(p1, p2) };
98+
assert_eq!(
99+
sign(fl),
100+
sign(gl),
101+
"strcasecmp a1={align1} a2={align2} len={len} base={base:?} v={v:?}: fl={fl} gl={gl}"
102+
);
103+
104+
for &n in &[0usize, 1, len / 2, len, len + 1, len + 8] {
105+
let fln = unsafe { fl_strncasecmp(p1, p2, n) };
106+
let gln = unsafe { libc::strncasecmp(p1, p2, n) };
107+
assert_eq!(
108+
sign(fln),
109+
sign(gln),
110+
"strncasecmp a1={align1} a2={align2} len={len} n={n}: fl={fln} gl={gln}"
111+
);
112+
checked += 1;
113+
}
114+
checked += 1;
115+
}
116+
}
117+
}
118+
}
119+
assert!(checked > 100_000, "corpus unexpectedly small: {checked}");
120+
}
121+
122+
#[test]
123+
fn strcasecmp_does_not_overread_past_guard_page() {
124+
let page = 4096usize;
125+
unsafe {
126+
let base = libc::mmap(
127+
std::ptr::null_mut(),
128+
page * 2,
129+
libc::PROT_READ | libc::PROT_WRITE,
130+
libc::MAP_PRIVATE | libc::MAP_ANONYMOUS,
131+
-1,
132+
0,
133+
);
134+
assert_ne!(base, libc::MAP_FAILED, "mmap failed");
135+
let base = base.cast::<u8>();
136+
assert_eq!(
137+
libc::mprotect(base.add(page).cast(), page, libc::PROT_NONE),
138+
0,
139+
"mprotect failed"
140+
);
141+
for nul_back in 1..=16usize {
142+
let start = base.add(page - nul_back);
143+
for k in 0..(nul_back - 1) {
144+
*start.add(k) = b'A';
145+
}
146+
*start.add(nul_back - 1) = 0;
147+
let other = b"aaaaaaaaaaaaaaaaaaaa\0".as_ptr().cast::<c_char>();
148+
let sp = start.cast::<c_char>();
149+
let _ = fl_strcasecmp(sp, other);
150+
let _ = fl_strcasecmp(other, sp);
151+
let _ = fl_strncasecmp(sp, other, 4096);
152+
let _ = fl_strncasecmp(other, sp, 4096);
153+
}
154+
libc::munmap(base.cast(), page * 2);
155+
}
156+
}

crates/frankenlibc-bench/benches/memset_abi_bench.rs

Lines changed: 54 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,9 +12,32 @@ use std::time::Instant;
1212

1313
use frankenlibc_abi::string_abi::{
1414
bench_raw_memcpy_bytes, bench_raw_memmove_bytes, bench_raw_memset_bytes, bench_scan_c_string,
15-
bench_scan_c_string_for_byte, bench_scan_c_string_last_byte, bench_scan_strcmp,
15+
bench_scan_c_string_for_byte, bench_scan_c_string_last_byte, bench_scan_strcasecmp,
16+
bench_scan_strcmp,
1617
};
1718

19+
/// Pre-lever strncasecmp scan: byte-at-a-time tolower compare to first diff/NUL.
20+
#[inline(never)]
21+
unsafe fn old_byte_strcasecmp(
22+
p1: *const std::os::raw::c_char,
23+
p2: *const std::os::raw::c_char,
24+
) -> i32 {
25+
unsafe {
26+
let mut i = 0usize;
27+
loop {
28+
let a = (*p1.add(i) as u8).to_ascii_lowercase();
29+
let b = (*p2.add(i) as u8).to_ascii_lowercase();
30+
if a != b {
31+
return (a as i32) - (b as i32);
32+
}
33+
if a == 0 {
34+
return 0;
35+
}
36+
i += 1;
37+
}
38+
}
39+
}
40+
1841
/// Pre-lever strrchr scan: byte-at-a-time, tracking the last target before NUL.
1942
#[inline(never)]
2043
unsafe fn old_byte_strrchr(p: *const std::os::raw::c_char, target: u8) -> Option<usize> {
@@ -327,4 +350,34 @@ fn main() {
327350
n, old, new, gl, old / new, gl / new,
328351
);
329352
}
353+
354+
println!("\nstrcasecmp (equal mod case → full scan, behind strcasecmp/strncasecmp):");
355+
println!(
356+
"{:>8} | {:>12} | {:>12} | {:>12} | {:>10} | {:>10}",
357+
"len", "old(ns)", "new(ns)", "glibc(ns)", "self x", "vs glibc"
358+
);
359+
for &n in &sizes {
360+
let mut a = vec![0x41u8; n + 1]; // 'A'*n
361+
a[n] = 0;
362+
let mut b = vec![0x61u8; n + 1]; // 'a'*n (equal mod case)
363+
b[n] = 0;
364+
let pa = a.as_ptr().cast::<std::os::raw::c_char>();
365+
let pb = b.as_ptr().cast::<std::os::raw::c_char>();
366+
let iters = (4_000_000u64 / (n as u64 + 1)).max(2000);
367+
368+
let old = median_ns_per_op(rounds, iters, || {
369+
black_box(unsafe { old_byte_strcasecmp(pa, pb) });
370+
});
371+
let new = median_ns_per_op(rounds, iters, || {
372+
black_box(unsafe { bench_scan_strcasecmp(pa, pb, usize::MAX) });
373+
});
374+
let gl = median_ns_per_op(rounds, iters, || {
375+
// SAFETY: both NUL-terminated.
376+
black_box(unsafe { libc::strcasecmp(pa, pb) });
377+
});
378+
println!(
379+
"{:>8} | {:>12.1} | {:>12.1} | {:>12.1} | {:>9.2}x | {:>9.2}x",
380+
n, old, new, gl, old / new, gl / new,
381+
);
382+
}
330383
}

0 commit comments

Comments
 (0)