Skip to content

Commit

Permalink
arm: Use __kuser_memory_barrier by default on ARMv6 Linux/Android
Browse files Browse the repository at this point in the history
  • Loading branch information
@taiki-e
taiki-e committed Sep 25, 2023
1 parent a13942e commit 0ebd6ec
Show file tree
Hide file tree
Showing 3 changed files with 108 additions and 12 deletions.
6 changes: 6 additions & 0 deletions .github/workflows/ci.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -250,6 +250,12 @@ jobs:
RUSTFLAGS: ${{ env.RUSTFLAGS }} -C target-feature=+v8
RUSTDOCFLAGS: ${{ env.RUSTDOCFLAGS }} -C target-feature=+v8
if: matrix.target == 'armv7-unknown-linux-gnueabi'
# arm v6 with legacy cp15_barrier
- run: tools/test.sh -vv --tests $TARGET $BUILD_STD $RELEASE
env:
RUSTFLAGS: ${{ env.RUSTFLAGS }} --cfg atomic_maybe_uninit_use_cp15_barrier
RUSTDOCFLAGS: ${{ env.RUSTDOCFLAGS }} --cfg atomic_maybe_uninit_use_cp15_barrier
if: matrix.target == 'arm-unknown-linux-gnueabi'
# arm v7 big endian
# armeb-unknown-linux-gnueabi is v8 by default, use custom target instead
- run: tools/test.sh -vv --tests --target armebv7-unknown-linux-gnueabi -Z build-std $RELEASE
Expand Down
59 changes: 52 additions & 7 deletions src/arch/arm.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,8 @@
// - armv7-a https://godbolt.org/z/Ws6o5ff6x
// - armv7-r https://godbolt.org/z/EasKf9c3q
// - armv7-m https://godbolt.org/z/Ef3hbfx7f
// - armv6 https://godbolt.org/z/ErPefoncM
// - armv6 (cp15_barrier) https://godbolt.org/z/EMvbbTcGx
// - armv6 (__kuser_memory_barrier) https://godbolt.org/z/83z7oxxoh
// - armv6-m https://godbolt.org/z/7nnaEq5zP

use core::{
Expand All @@ -36,23 +37,41 @@ macro_rules! dmb {
"dmb ish"
};
}
// ARMv6 does not support `dmb ish`, so use use special instruction equivalent to a DMB.
// Only a full system barrier exists in the M-class architectures.
#[cfg(any(target_feature = "mclass", atomic_maybe_uninit_target_feature = "mclass"))]
macro_rules! dmb {
() => {
"dmb sy"
};
}
// ARMv6 does not support `dmb`, so use use special instruction equivalent to a DMB.
//
// Refs:
// - https://reviews.llvm.org/D5386
// - https://developer.arm.com/documentation/ddi0360/e/control-coprocessor-cp15/register-descriptions/c7--cache-operations-register?lang=en
#[cfg(not(all(
any(target_os = "linux", target_os = "android"),
not(atomic_maybe_uninit_use_cp15_barrier),
)))]
#[cfg(not(any(target_feature = "v7", atomic_maybe_uninit_target_feature = "v7")))]
#[cfg(not(any(target_feature = "mclass", atomic_maybe_uninit_target_feature = "mclass")))]
macro_rules! dmb {
() => {
"mcr p15, #0, {zero}, c7, c10, #5"
};
}
// Only a full system barrier exists in the M-class architectures.
#[cfg(any(target_feature = "mclass", atomic_maybe_uninit_target_feature = "mclass"))]
// We prefer __kuser_memory_barrier over cp15_barrier because cp15_barrier is
// trapped and emulated by default on Linux/Android with ARMv8+ (or ARMv7+?).
// https://github.com/rust-lang/rust/issues/60605
#[cfg(all(
any(target_os = "linux", target_os = "android"),
not(atomic_maybe_uninit_use_cp15_barrier),
))]
#[cfg(not(any(target_feature = "v7", atomic_maybe_uninit_target_feature = "v7")))]
#[cfg(not(any(target_feature = "mclass", atomic_maybe_uninit_target_feature = "mclass")))]
macro_rules! dmb {
() => {
"dmb sy"
"blx {kuser_memory_barrier}"
};
}

Expand All @@ -77,7 +96,6 @@ macro_rules! clrex {
};
}

// On ARMv6, dmb! calls `mcr p15, 0, <Rd>, c7, c10, 5`, and the value in the Rd register should be zero (SBZ).
macro_rules! asm_no_dmb {
($($asm:tt)*) => {
core::arch::asm!($($asm)*)
Expand All @@ -94,18 +112,45 @@ macro_rules! asm_use_dmb {
core::arch::asm!($($asm)*)
};
}
#[cfg(not(all(
any(target_os = "linux", target_os = "android"),
not(atomic_maybe_uninit_use_cp15_barrier),
)))]
#[cfg(not(any(
target_feature = "v7",
atomic_maybe_uninit_target_feature = "v7",
target_feature = "mclass",
atomic_maybe_uninit_target_feature = "mclass",
)))]
macro_rules! asm_use_dmb {
($($asm:tt)*) => {
($($asm:tt)*) => {{
// In this case, dmb! calls `mcr p15, 0, <Rd>, c7, c10, 5`, and the value in the Rd register should be zero (SBZ).
core::arch::asm!(
$($asm)*
zero = inout(reg) 0_u32 => _,
)
}};
}
#[cfg(all(
any(target_os = "linux", target_os = "android"),
not(atomic_maybe_uninit_use_cp15_barrier),
))]
#[cfg(not(any(
target_feature = "v7",
atomic_maybe_uninit_target_feature = "v7",
target_feature = "mclass",
atomic_maybe_uninit_target_feature = "mclass",
)))]
macro_rules! asm_use_dmb {
($($asm:tt)*) => {
// In this case, dmb! calls __kuser_memory_barrier.
core::arch::asm!(
$($asm)*
// __kuser_memory_barrier (see also arm_linux.rs)
// https://www.kernel.org/doc/Documentation/arm/kernel_user_helpers.txt
kuser_memory_barrier = inout(reg) 0xFFFF0FA0_usize => _,
out("lr") _,
)
};
}

Expand Down
55 changes: 50 additions & 5 deletions src/arch_legacy/arm.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,23 +34,41 @@ macro_rules! dmb {
"dmb ish"
};
}
// ARMv6 does not support `dmb ish`, so use use special instruction equivalent to a DMB.
// Only a full system barrier exists in the M-class architectures.
#[cfg(any(target_feature = "mclass", atomic_maybe_uninit_target_feature = "mclass"))]
macro_rules! dmb {
() => {
"dmb sy"
};
}
// ARMv6 does not support `dmb`, so use use special instruction equivalent to a DMB.
//
// Refs:
// - https://reviews.llvm.org/D5386
// - https://developer.arm.com/documentation/ddi0360/e/control-coprocessor-cp15/register-descriptions/c7--cache-operations-register?lang=en
#[cfg(not(all(
any(target_os = "linux", target_os = "android"),
not(atomic_maybe_uninit_use_cp15_barrier),
)))]
#[cfg(not(any(target_feature = "v7", atomic_maybe_uninit_target_feature = "v7")))]
#[cfg(not(any(target_feature = "mclass", atomic_maybe_uninit_target_feature = "mclass")))]
macro_rules! dmb {
() => {
"mcr p15, #0, r0, c7, c10, #5"
};
}
// Only a full system barrier exists in the M-class architectures.
#[cfg(any(target_feature = "mclass", atomic_maybe_uninit_target_feature = "mclass"))]
// We prefer __kuser_memory_barrier over cp15_barrier because cp15_barrier is
// trapped and emulated by default on Linux/Android with ARMv8+ (or ARMv7+?).
// https://github.com/rust-lang/rust/issues/60605
#[cfg(all(
any(target_os = "linux", target_os = "android"),
not(atomic_maybe_uninit_use_cp15_barrier),
))]
#[cfg(not(any(target_feature = "v7", atomic_maybe_uninit_target_feature = "v7")))]
#[cfg(not(any(target_feature = "mclass", atomic_maybe_uninit_target_feature = "mclass")))]
macro_rules! dmb {
() => {
"dmb sy"
"blx {kuser_memory_barrier}"
};
}

Expand All @@ -75,7 +93,6 @@ macro_rules! clrex {
};
}

// On ARMv6, dmb! refers to r0, so when calling it, we must clobbering r0.
macro_rules! asm_no_dmb {
(options($($options:tt)*), $($asm:tt)*) => {
core::arch::asm!(
Expand All @@ -98,6 +115,10 @@ macro_rules! asm_use_dmb {
)
};
}
#[cfg(not(all(
any(target_os = "linux", target_os = "android"),
not(atomic_maybe_uninit_use_cp15_barrier),
)))]
#[cfg(not(any(
target_feature = "v7",
atomic_maybe_uninit_target_feature = "v7",
Expand All @@ -106,13 +127,37 @@ macro_rules! asm_use_dmb {
)))]
macro_rules! asm_use_dmb {
(options($($options:tt)*), $($asm:tt)*) => {
// In this case, dmb! calls `mcr p15, 0, <Rd>, c7, c10, 5`, and the value in the Rd register should be zero (SBZ).
core::arch::asm!(
$($asm)*
inout("r0") 0_u32 => _,
options($($options)*),
)
};
}
#[cfg(all(
any(target_os = "linux", target_os = "android"),
not(atomic_maybe_uninit_use_cp15_barrier),
))]
#[cfg(not(any(
target_feature = "v7",
atomic_maybe_uninit_target_feature = "v7",
target_feature = "mclass",
atomic_maybe_uninit_target_feature = "mclass",
)))]
macro_rules! asm_use_dmb {
(options($($options:tt)*), $($asm:tt)*) => {
// In this case, dmb! calls __kuser_memory_barrier.
core::arch::asm!(
$($asm)*
// __kuser_memory_barrier (see also arm_linux.rs)
// https://www.kernel.org/doc/Documentation/arm/kernel_user_helpers.txt
inout("r0") 0xFFFF0FA0_usize => _,
out("lr") _,
options($($options)*),
)
};
}

macro_rules! atomic {
($int_type:ident, $asm_suffix:tt) => {
Expand Down

0 comments on commit 0ebd6ec

Please sign in to comment.