Skip to content

Commit 650815d

Browse files
ShnatselShnatsel
authored
Replace remaining NEON unsafe loads with checked_transmute_copy (#237)
Follow-up to #233 which did this for x86 Leaves `vld4q_*` untouched because those are deinterleaving loads that genuinely need an intrinsic.
1 parent 0d13b0a commit 650815d

3 files changed

Lines changed: 55 additions & 22 deletions

File tree

‎fearless_simd/src/generated/neon.rs‎

Lines changed: 18 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -780,7 +780,9 @@ impl Simd for Neon {
780780
#[inline(always)]
781781
fn from_bitmask_mask8x16(self, bits: u64) -> mask8x16<Self> {
782782
unsafe {
783-
let shifts = vld1q_s16([15, 14, 13, 12, 11, 10, 9, 8].as_ptr());
783+
let shifts = crate::transmute::checked_transmute_copy::<[i16; 8], int16x8_t>(&[
784+
15, 14, 13, 12, 11, 10, 9, 8,
785+
]);
784786
let lo = vshlq_u16(vdupq_n_u16(bits as u16), shifts);
785787
let hi = vshlq_u16(vdupq_n_u16((bits >> 8) as u16), shifts);
786788
let lo = vcltq_s16(vreinterpretq_s16_u16(lo), vdupq_n_s16(0));
@@ -795,8 +797,9 @@ impl Simd for Neon {
795797
#[inline(always)]
796798
fn to_bitmask_mask8x16(self, a: mask8x16<Self>) -> u64 {
797799
unsafe {
798-
let weights =
799-
vld1q_u8([1, 2, 4, 8, 16, 32, 64, 128, 1, 2, 4, 8, 16, 32, 64, 128].as_ptr());
800+
let weights = crate::transmute::checked_transmute_copy::<[u8; 16], uint8x16_t>(&[
801+
1, 2, 4, 8, 16, 32, 64, 128, 1, 2, 4, 8, 16, 32, 64, 128,
802+
]);
800803
let bits = vandq_u8(vreinterpretq_u8_s8(a.into()), weights);
801804
let lo = vaddv_u8(vget_low_u8(bits)) as u64;
802805
let hi = vaddv_u8(vget_high_u8(bits)) as u64;
@@ -1284,7 +1287,9 @@ impl Simd for Neon {
12841287
#[inline(always)]
12851288
fn from_bitmask_mask16x8(self, bits: u64) -> mask16x8<Self> {
12861289
unsafe {
1287-
let shifts = vld1q_s16([15, 14, 13, 12, 11, 10, 9, 8].as_ptr());
1290+
let shifts = crate::transmute::checked_transmute_copy::<[i16; 8], int16x8_t>(&[
1291+
15, 14, 13, 12, 11, 10, 9, 8,
1292+
]);
12881293
let shifted = vshlq_u16(vdupq_n_u16(bits as u16), shifts);
12891294
let mask = vcltq_s16(vreinterpretq_s16_u16(shifted), vdupq_n_s16(0));
12901295
vreinterpretq_s16_u16(mask).simd_into(self)
@@ -1293,7 +1298,9 @@ impl Simd for Neon {
12931298
#[inline(always)]
12941299
fn to_bitmask_mask16x8(self, a: mask16x8<Self>) -> u64 {
12951300
unsafe {
1296-
let weights = vld1q_u16([1, 2, 4, 8, 16, 32, 64, 128].as_ptr());
1301+
let weights = crate::transmute::checked_transmute_copy::<[u16; 8], uint16x8_t>(&[
1302+
1, 2, 4, 8, 16, 32, 64, 128,
1303+
]);
12971304
let bits = vandq_u16(vreinterpretq_u16_s16(a.into()), weights);
12981305
vaddvq_u16(bits) as u64
12991306
}
@@ -1783,7 +1790,8 @@ impl Simd for Neon {
17831790
#[inline(always)]
17841791
fn from_bitmask_mask32x4(self, bits: u64) -> mask32x4<Self> {
17851792
unsafe {
1786-
let shifts = vld1q_s32([31, 30, 29, 28].as_ptr());
1793+
let shifts =
1794+
crate::transmute::checked_transmute_copy::<[i32; 4], int32x4_t>(&[31, 30, 29, 28]);
17871795
let shifted = vshlq_u32(vdupq_n_u32(bits as u32), shifts);
17881796
let mask = vcltq_s32(vreinterpretq_s32_u32(shifted), vdupq_n_s32(0));
17891797
vreinterpretq_s32_u32(mask).simd_into(self)
@@ -1792,7 +1800,8 @@ impl Simd for Neon {
17921800
#[inline(always)]
17931801
fn to_bitmask_mask32x4(self, a: mask32x4<Self>) -> u64 {
17941802
unsafe {
1795-
let weights = vld1q_u32([1, 2, 4, 8].as_ptr());
1803+
let weights =
1804+
crate::transmute::checked_transmute_copy::<[u32; 4], uint32x4_t>(&[1, 2, 4, 8]);
17961805
let bits = vandq_u32(vreinterpretq_u32_s32(a.into()), weights);
17971806
vaddvq_u32(bits) as u64
17981807
}
@@ -2103,7 +2112,7 @@ impl Simd for Neon {
21032112
#[inline(always)]
21042113
fn from_bitmask_mask64x2(self, bits: u64) -> mask64x2<Self> {
21052114
unsafe {
2106-
let shifts = vld1q_s64([63, 62].as_ptr());
2115+
let shifts = crate::transmute::checked_transmute_copy::<[i64; 2], int64x2_t>(&[63, 62]);
21072116
let shifted = vshlq_u64(vdupq_n_u64(bits), shifts);
21082117
let mask = vcltq_s64(vreinterpretq_s64_u64(shifted), vdupq_n_s64(0));
21092118
vreinterpretq_s64_u64(mask).simd_into(self)
@@ -2112,7 +2121,7 @@ impl Simd for Neon {
21122121
#[inline(always)]
21132122
fn to_bitmask_mask64x2(self, a: mask64x2<Self>) -> u64 {
21142123
unsafe {
2115-
let weights = vld1q_u64([1, 2].as_ptr());
2124+
let weights = crate::transmute::checked_transmute_copy::<[u64; 2], uint64x2_t>(&[1, 2]);
21162125
let bits = vandq_u64(vreinterpretq_u64_s64(a.into()), weights);
21172126
vaddvq_u64(bits)
21182127
}

‎fearless_simd/src/transmute.rs‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@ use core::arch::aarch64::{
1616
int8x16_t, int8x16x2_t, int8x16x4_t, int16x8_t, int16x8x2_t, int16x8x4_t, int32x4_t,
1717
int32x4x2_t, int32x4x4_t, int64x2_t, int64x2x2_t, int64x2x4_t, uint8x16_t, uint8x16x2_t,
1818
uint8x16x4_t, uint16x8_t, uint16x8x2_t, uint16x8x4_t, uint32x4_t, uint32x4x2_t, uint32x4x4_t,
19+
uint64x2_t,
1920
};
2021
#[cfg(all(target_arch = "wasm32", target_feature = "simd128"))]
2122
use core::arch::wasm32::v128;
@@ -160,6 +161,7 @@ const _: () = {
160161
unsafe impl SimdPod for uint32x4_t {}
161162
unsafe impl SimdPod for uint32x4x2_t {}
162163
unsafe impl SimdPod for uint32x4x4_t {}
164+
unsafe impl SimdPod for uint64x2_t {}
163165
};
164166

165167
#[cfg(any(target_arch = "x86", target_arch = "x86_64"))]

‎fearless_simd_gen/src/mk_neon.rs‎

Lines changed: 35 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -580,7 +580,10 @@ impl Neon {
580580
8 => quote! {
581581
#method_sig {
582582
unsafe {
583-
let shifts = vld1q_s16([15, 14, 13, 12, 11, 10, 9, 8].as_ptr());
583+
let shifts =
584+
crate::transmute::checked_transmute_copy::<[i16; 8], int16x8_t>(
585+
&[15, 14, 13, 12, 11, 10, 9, 8],
586+
);
584587
let lo = vshlq_u16(vdupq_n_u16(bits as u16), shifts);
585588
let hi = vshlq_u16(vdupq_n_u16((bits >> 8) as u16), shifts);
586589
let lo = vcltq_s16(vreinterpretq_s16_u16(lo), vdupq_n_s16(0));
@@ -595,7 +598,10 @@ impl Neon {
595598
16 => quote! {
596599
#method_sig {
597600
unsafe {
598-
let shifts = vld1q_s16([15, 14, 13, 12, 11, 10, 9, 8].as_ptr());
601+
let shifts =
602+
crate::transmute::checked_transmute_copy::<[i16; 8], int16x8_t>(
603+
&[15, 14, 13, 12, 11, 10, 9, 8],
604+
);
599605
let shifted = vshlq_u16(vdupq_n_u16(bits as u16), shifts);
600606
let mask = vcltq_s16(vreinterpretq_s16_u16(shifted), vdupq_n_s16(0));
601607
vreinterpretq_s16_u16(mask).simd_into(self)
@@ -605,7 +611,10 @@ impl Neon {
605611
32 => quote! {
606612
#method_sig {
607613
unsafe {
608-
let shifts = vld1q_s32([31, 30, 29, 28].as_ptr());
614+
let shifts =
615+
crate::transmute::checked_transmute_copy::<[i32; 4], int32x4_t>(
616+
&[31, 30, 29, 28],
617+
);
609618
let shifted = vshlq_u32(vdupq_n_u32(bits as u32), shifts);
610619
let mask = vcltq_s32(vreinterpretq_s32_u32(shifted), vdupq_n_s32(0));
611620
vreinterpretq_s32_u32(mask).simd_into(self)
@@ -615,7 +624,10 @@ impl Neon {
615624
64 => quote! {
616625
#method_sig {
617626
unsafe {
618-
let shifts = vld1q_s64([63, 62].as_ptr());
627+
let shifts =
628+
crate::transmute::checked_transmute_copy::<[i64; 2], int64x2_t>(
629+
&[63, 62],
630+
);
619631
let shifted = vshlq_u64(vdupq_n_u64(bits), shifts);
620632
let mask = vcltq_s64(vreinterpretq_s64_u64(shifted), vdupq_n_s64(0));
621633
vreinterpretq_s64_u64(mask).simd_into(self)
@@ -642,10 +654,13 @@ impl Neon {
642654
8 => quote! {
643655
#method_sig {
644656
unsafe {
645-
let weights = vld1q_u8([
646-
1, 2, 4, 8, 16, 32, 64, 128,
647-
1, 2, 4, 8, 16, 32, 64, 128,
648-
].as_ptr());
657+
let weights =
658+
crate::transmute::checked_transmute_copy::<[u8; 16], uint8x16_t>(
659+
&[
660+
1, 2, 4, 8, 16, 32, 64, 128,
661+
1, 2, 4, 8, 16, 32, 64, 128,
662+
],
663+
);
649664
let bits = vandq_u8(vreinterpretq_u8_s8(a.into()), weights);
650665
let lo = vaddv_u8(vget_low_u8(bits)) as u64;
651666
let hi = vaddv_u8(vget_high_u8(bits)) as u64;
@@ -656,9 +671,10 @@ impl Neon {
656671
16 => quote! {
657672
#method_sig {
658673
unsafe {
659-
let weights = vld1q_u16([
660-
1, 2, 4, 8, 16, 32, 64, 128,
661-
].as_ptr());
674+
let weights =
675+
crate::transmute::checked_transmute_copy::<[u16; 8], uint16x8_t>(
676+
&[1, 2, 4, 8, 16, 32, 64, 128],
677+
);
662678
let bits = vandq_u16(vreinterpretq_u16_s16(a.into()), weights);
663679
vaddvq_u16(bits) as u64
664680
}
@@ -667,7 +683,10 @@ impl Neon {
667683
32 => quote! {
668684
#method_sig {
669685
unsafe {
670-
let weights = vld1q_u32([1, 2, 4, 8].as_ptr());
686+
let weights =
687+
crate::transmute::checked_transmute_copy::<[u32; 4], uint32x4_t>(
688+
&[1, 2, 4, 8],
689+
);
671690
let bits = vandq_u32(vreinterpretq_u32_s32(a.into()), weights);
672691
vaddvq_u32(bits) as u64
673692
}
@@ -676,7 +695,10 @@ impl Neon {
676695
64 => quote! {
677696
#method_sig {
678697
unsafe {
679-
let weights = vld1q_u64([1, 2].as_ptr());
698+
let weights =
699+
crate::transmute::checked_transmute_copy::<[u64; 2], uint64x2_t>(
700+
&[1, 2],
701+
);
680702
let bits = vandq_u64(vreinterpretq_u64_s64(a.into()), weights);
681703
vaddvq_u64(bits)
682704
}

0 commit comments

Comments
 (0)