From c9f7324c7407ccdde32a8caad55b54b81572e8ef Mon Sep 17 00:00:00 2001 From: joboet Date: Sat, 9 Mar 2024 12:46:11 +0100 Subject: [PATCH 1/2] std: allow `weak` in item position on Apple platforms --- library/std/src/sys/pal/unix/fd.rs | 8 ++++---- library/std/src/sys/pal/unix/weak.rs | 11 ++++++----- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/library/std/src/sys/pal/unix/fd.rs b/library/std/src/sys/pal/unix/fd.rs index 2fc33bdfefbf5..7bc7bcb708752 100644 --- a/library/std/src/sys/pal/unix/fd.rs +++ b/library/std/src/sys/pal/unix/fd.rs @@ -284,9 +284,9 @@ impl FileDesc { super::weak::weak!(fn preadv(libc::c_int, *const libc::iovec, libc::c_int, off64_t) -> isize); match preadv.get() { - Some(preadv) => { + Some(read) => { let ret = cvt(unsafe { - preadv( + read( self.as_raw_fd(), bufs.as_mut_ptr() as *mut libc::iovec as *const libc::iovec, cmp::min(bufs.len(), max_iov()) as libc::c_int, @@ -477,9 +477,9 @@ impl FileDesc { super::weak::weak!(fn pwritev(libc::c_int, *const libc::iovec, libc::c_int, off64_t) -> isize); match pwritev.get() { - Some(pwritev) => { + Some(read) => { let ret = cvt(unsafe { - pwritev( + read( self.as_raw_fd(), bufs.as_ptr() as *const libc::iovec, cmp::min(bufs.len(), max_iov()) as libc::c_int, diff --git a/library/std/src/sys/pal/unix/weak.rs b/library/std/src/sys/pal/unix/weak.rs index 35762f5a53b5b..21971077bb263 100644 --- a/library/std/src/sys/pal/unix/weak.rs +++ b/library/std/src/sys/pal/unix/weak.rs @@ -62,15 +62,16 @@ impl ExternWeak { } pub(crate) macro dlsym { - (fn $name:ident($($t:ty),*) -> $ret:ty) => ( - dlsym!(fn $name($($t),*) -> $ret, stringify!($name)); + ($v:vis fn $name:ident($($t:ty),*) -> $ret:ty) => ( + dlsym!($v fn $name($($t),*) -> $ret, stringify!($name)); ), - (fn $name:ident($($t:ty),*) -> $ret:ty, $sym:expr) => ( - static DLSYM: DlsymWeak $ret> = + ($v:vis fn $name:ident($($t:ty),*) -> $ret:ty, $sym:expr) => ( + #[allow(non_upper_case_globals)] + $v static $name: DlsymWeak $ret> = DlsymWeak::new(concat!($sym, '\0')); - let $name = &DLSYM; ) } + pub(crate) struct DlsymWeak { name: &'static str, func: AtomicPtr, From dc49e21d213ef0539f017268d8f8051d19deac6a Mon Sep 17 00:00:00 2001 From: joboet Date: Mon, 3 Feb 2025 16:46:22 +0100 Subject: [PATCH 2/2] std: use futex for everything but `Mutex` on Apple platforms --- library/std/src/sys/pal/unix/futex.rs | 184 ++++++++++++++++++ library/std/src/sys/pal/unix/sync/mod.rs | 2 + library/std/src/sys/sync/condvar/mod.rs | 1 + library/std/src/sys/sync/once/mod.rs | 4 + library/std/src/sys/sync/rwlock/mod.rs | 1 + .../std/src/sys/sync/thread_parking/darwin.rs | 130 ------------- .../std/src/sys/sync/thread_parking/mod.rs | 1 + 7 files changed, 193 insertions(+), 130 deletions(-) delete mode 100644 library/std/src/sys/sync/thread_parking/darwin.rs diff --git a/library/std/src/sys/pal/unix/futex.rs b/library/std/src/sys/pal/unix/futex.rs index 0fc765dc87a5d..a81c8080516f0 100644 --- a/library/std/src/sys/pal/unix/futex.rs +++ b/library/std/src/sys/pal/unix/futex.rs @@ -1,5 +1,6 @@ #![cfg(any( target_os = "linux", + target_vendor = "apple", target_os = "android", all(target_os = "emscripten", target_feature = "atomics"), target_os = "freebsd", @@ -145,6 +146,189 @@ pub fn futex_wake_all(futex: &AtomicU32) { }; } +/// With macOS version 14.4, Apple introduced a public futex API. Unfortunately, +/// our minimum supported version is 10.12, so we need a fallback API. Luckily +/// for us, the underlying syscalls have been available since exactly that +/// version, so we just use those when needed. This is private API however, +/// which means we need to take care to avoid breakage if the syscall is removed +/// and to avoid apps being rejected from the App Store. To do this, we use weak +/// linkage emulation for both the public and the private API. +/// +/// See https://developer.apple.com/documentation/os/os_sync_wait_on_address?language=objc +/// for documentation of the public API and +/// https://github.com/apple-oss-distributions/xnu/blob/1031c584a5e37aff177559b9f69dbd3c8c3fd30a/bsd/sys/ulock.h#L69 +/// for the header file of the private API, along with its usage in libpthread +/// https://github.com/apple-oss-distributions/libpthread/blob/d8c4e3c212553d3e0f5d76bb7d45a8acd61302dc/src/pthread_cond.c#L463 +#[cfg(target_vendor = "apple")] +mod apple { + use crate::ffi::{c_int, c_void}; + // Explicitly use `dlsym` for the private fallback API to ensure that the + // App Store checks do not flag Apps using Rust as relying on private API. + use crate::sys::pal::weak::{dlsym, weak}; + + pub const OS_CLOCK_MACH_ABSOLUTE_TIME: u32 = 32; + pub const OS_SYNC_WAIT_ON_ADDRESS_NONE: u32 = 0; + pub const OS_SYNC_WAKE_BY_ADDRESS_NONE: u32 = 0; + + pub const UL_COMPARE_AND_WAIT: u32 = 1; + pub const ULF_WAKE_ALL: u32 = 0x100; + // The syscalls support directly returning errors instead of going through errno. + pub const ULF_NO_ERRNO: u32 = 0x1000000; + + // These functions appeared with macOS 14.4, iOS 17.4, tvOS 17.4, watchOS 10.4, visionOS 1.1. + weak! { + pub fn os_sync_wait_on_address(*mut c_void, u64, usize, u32) -> c_int + } + + weak! { + pub fn os_sync_wait_on_address_with_timeout(*mut c_void, u64, usize, u32, u32, u64) -> c_int + } + + weak! { + pub fn os_sync_wake_by_address_any(*mut c_void, usize, u32) -> c_int + } + + weak! { + pub fn os_sync_wake_by_address_all(*mut c_void, usize, u32) -> c_int + } + + // This syscall appeared with macOS 11.0. + // It is used to support nanosecond precision for timeouts, among other features. + dlsym! { + pub fn __ulock_wait2(u32, *mut c_void, u64, u64, u64) -> c_int + } + + // These syscalls appeared with macOS 10.12. + dlsym! { + pub fn __ulock_wait(u32, *mut c_void, u64, u32) -> c_int + } + + dlsym! { + pub fn __ulock_wake(u32, *mut c_void, u64) -> c_int + } +} + +#[cfg(target_vendor = "apple")] +pub fn futex_wait(futex: &AtomicU32, expected: u32, timeout: Option) -> bool { + use apple::*; + + use crate::mem::size_of; + + let addr = futex.as_ptr().cast(); + let value = expected as u64; + let size = size_of::(); + if let Some(timeout) = timeout { + let timeout_ns = timeout.as_nanos().clamp(1, u64::MAX as u128) as u64; + let timeout_ms = timeout.as_micros().clamp(1, u32::MAX as u128) as u32; + + if let Some(wait) = os_sync_wait_on_address_with_timeout.get() { + let r = unsafe { + wait( + addr, + value, + size, + OS_SYNC_WAIT_ON_ADDRESS_NONE, + OS_CLOCK_MACH_ABSOLUTE_TIME, + timeout_ns, + ) + }; + + // We promote spurious wakeups (reported as EINTR) to normal ones for + // simplicity. + r != -1 || super::os::errno() != libc::ETIMEDOUT + } else if let Some(wait) = __ulock_wait2.get() { + unsafe { + wait(UL_COMPARE_AND_WAIT | ULF_NO_ERRNO, addr, value, timeout_ns, 0) + != -libc::ETIMEDOUT + } + } else if let Some(wait) = __ulock_wait.get() { + unsafe { + wait(UL_COMPARE_AND_WAIT | ULF_NO_ERRNO, addr, value, timeout_ms) + != -libc::ETIMEDOUT + } + } else { + panic!("your system is below the minimum supported version of Rust"); + } + } else { + if let Some(wait) = os_sync_wait_on_address.get() { + unsafe { + wait(addr, value, size, OS_SYNC_WAIT_ON_ADDRESS_NONE); + } + } else if let Some(wait) = __ulock_wait.get() { + unsafe { + wait(UL_COMPARE_AND_WAIT | ULF_NO_ERRNO, addr, value, 0); + } + } else { + panic!("your system is below the minimum supported version of Rust"); + } + true + } +} + +#[cfg(target_vendor = "apple")] +pub fn futex_wake(futex: &AtomicU32) -> bool { + use apple::*; + + use crate::io::Error; + use crate::mem::size_of; + + let addr = futex.as_ptr().cast(); + if let Some(wake) = os_sync_wake_by_address_any.get() { + unsafe { wake(addr, size_of::(), OS_SYNC_WAKE_BY_ADDRESS_NONE) == 0 } + } else if let Some(wake) = __ulock_wake.get() { + // Judging by its use in pthreads, __ulock_wake can get interrupted, so + // retry until either waking up a waiter or failing because there are no + // waiters (ENOENT). + loop { + let r = unsafe { wake(UL_COMPARE_AND_WAIT | ULF_NO_ERRNO, addr, 0) }; + + if r >= 0 { + return true; + } else { + match -r { + libc::ENOENT => return false, + libc::EINTR => continue, + err => panic!("__ulock_wake failed: {}", Error::from_raw_os_error(err)), + } + } + } + } else { + panic!("your system is below the minimum supported version of Rust"); + } +} + +#[cfg(target_vendor = "apple")] +pub fn futex_wake_all(futex: &AtomicU32) { + use apple::*; + + use crate::io::Error; + use crate::mem::size_of; + + let addr = futex.as_ptr().cast(); + + if let Some(wake) = os_sync_wake_by_address_all.get() { + unsafe { + wake(addr, size_of::(), OS_SYNC_WAKE_BY_ADDRESS_NONE); + } + } else if let Some(wake) = __ulock_wake.get() { + loop { + let r = unsafe { wake(UL_COMPARE_AND_WAIT | ULF_WAKE_ALL | ULF_NO_ERRNO, addr, 0) }; + + if r >= 0 { + return; + } else { + match -r { + libc::ENOENT => return, + libc::EINTR => continue, + err => panic!("__ulock_wake failed: {}", Error::from_raw_os_error(err)), + } + } + } + } else { + panic!("your system is below the minimum supported version of Rust"); + } +} + #[cfg(target_os = "openbsd")] pub fn futex_wait(futex: &AtomicU32, expected: u32, timeout: Option) -> bool { use super::time::Timespec; diff --git a/library/std/src/sys/pal/unix/sync/mod.rs b/library/std/src/sys/pal/unix/sync/mod.rs index b430ff5d8ef5f..17f5b50e14728 100644 --- a/library/std/src/sys/pal/unix/sync/mod.rs +++ b/library/std/src/sys/pal/unix/sync/mod.rs @@ -9,8 +9,10 @@ )))] #![forbid(unsafe_op_in_unsafe_fn)] +#[cfg(not(target_vendor = "apple"))] mod condvar; mod mutex; +#[cfg(not(target_vendor = "apple"))] pub use condvar::Condvar; pub use mutex::Mutex; diff --git a/library/std/src/sys/sync/condvar/mod.rs b/library/std/src/sys/sync/condvar/mod.rs index d0c998a559737..d38dc80142f36 100644 --- a/library/std/src/sys/sync/condvar/mod.rs +++ b/library/std/src/sys/sync/condvar/mod.rs @@ -2,6 +2,7 @@ cfg_if::cfg_if! { if #[cfg(any( all(target_os = "windows", not(target_vendor="win7")), target_os = "linux", + target_vendor = "apple", target_os = "android", target_os = "freebsd", target_os = "openbsd", diff --git a/library/std/src/sys/sync/once/mod.rs b/library/std/src/sys/sync/once/mod.rs index 0e38937b1219a..417ddb0d19c88 100644 --- a/library/std/src/sys/sync/once/mod.rs +++ b/library/std/src/sys/sync/once/mod.rs @@ -18,6 +18,10 @@ cfg_if::cfg_if! { target_os = "dragonfly", target_os = "fuchsia", target_os = "hermit", + target_os = "macos", + target_os = "ios", + target_os = "tvos", + target_os = "watchos", ))] { mod futex; pub use futex::{Once, OnceState}; diff --git a/library/std/src/sys/sync/rwlock/mod.rs b/library/std/src/sys/sync/rwlock/mod.rs index 70ba6bf38ef55..8e0259f03038c 100644 --- a/library/std/src/sys/sync/rwlock/mod.rs +++ b/library/std/src/sys/sync/rwlock/mod.rs @@ -2,6 +2,7 @@ cfg_if::cfg_if! { if #[cfg(any( all(target_os = "windows", not(target_vendor = "win7")), target_os = "linux", + target_vendor = "apple", target_os = "android", target_os = "freebsd", target_os = "openbsd", diff --git a/library/std/src/sys/sync/thread_parking/darwin.rs b/library/std/src/sys/sync/thread_parking/darwin.rs deleted file mode 100644 index 0553c5e19a91f..0000000000000 --- a/library/std/src/sys/sync/thread_parking/darwin.rs +++ /dev/null @@ -1,130 +0,0 @@ -//! Thread parking for Darwin-based systems. -//! -//! Darwin actually has futex syscalls (`__ulock_wait`/`__ulock_wake`), but they -//! cannot be used in `std` because they are non-public (their use will lead to -//! rejection from the App Store). -//! -//! Therefore, we need to look for other synchronization primitives. Luckily, Darwin -//! supports semaphores, which allow us to implement the behavior we need with -//! only one primitive (as opposed to a mutex-condvar pair). We use the semaphore -//! provided by libdispatch, as the underlying Mach semaphore is only dubiously -//! public. - -#![allow(non_camel_case_types)] - -use crate::pin::Pin; -use crate::sync::atomic::AtomicI8; -use crate::sync::atomic::Ordering::{Acquire, Release}; -use crate::time::Duration; - -type dispatch_semaphore_t = *mut crate::ffi::c_void; -type dispatch_time_t = u64; - -const DISPATCH_TIME_NOW: dispatch_time_t = 0; -const DISPATCH_TIME_FOREVER: dispatch_time_t = !0; - -// Contained in libSystem.dylib, which is linked by default. -extern "C" { - fn dispatch_time(when: dispatch_time_t, delta: i64) -> dispatch_time_t; - fn dispatch_semaphore_create(val: isize) -> dispatch_semaphore_t; - fn dispatch_semaphore_wait(dsema: dispatch_semaphore_t, timeout: dispatch_time_t) -> isize; - fn dispatch_semaphore_signal(dsema: dispatch_semaphore_t) -> isize; - fn dispatch_release(object: *mut crate::ffi::c_void); -} - -const EMPTY: i8 = 0; -const NOTIFIED: i8 = 1; -const PARKED: i8 = -1; - -pub struct Parker { - semaphore: dispatch_semaphore_t, - state: AtomicI8, -} - -unsafe impl Sync for Parker {} -unsafe impl Send for Parker {} - -impl Parker { - pub unsafe fn new_in_place(parker: *mut Parker) { - let semaphore = dispatch_semaphore_create(0); - assert!( - !semaphore.is_null(), - "failed to create dispatch semaphore for thread synchronization" - ); - parker.write(Parker { semaphore, state: AtomicI8::new(EMPTY) }) - } - - // Does not need `Pin`, but other implementation do. - pub unsafe fn park(self: Pin<&Self>) { - // The semaphore counter must be zero at this point, because unparking - // threads will not actually increase it until we signalled that we - // are waiting. - - // Change NOTIFIED to EMPTY and EMPTY to PARKED. - if self.state.fetch_sub(1, Acquire) == NOTIFIED { - return; - } - - // Another thread may increase the semaphore counter from this point on. - // If it is faster than us, we will decrement it again immediately below. - // If we are faster, we wait. - - // Ensure that the semaphore counter has actually been decremented, even - // if the call timed out for some reason. - while dispatch_semaphore_wait(self.semaphore, DISPATCH_TIME_FOREVER) != 0 {} - - // At this point, the semaphore counter is zero again. - - // We were definitely woken up, so we don't need to check the state. - // Still, we need to reset the state using a swap to observe the state - // change with acquire ordering. - self.state.swap(EMPTY, Acquire); - } - - // Does not need `Pin`, but other implementation do. - pub unsafe fn park_timeout(self: Pin<&Self>, dur: Duration) { - if self.state.fetch_sub(1, Acquire) == NOTIFIED { - return; - } - - let nanos = dur.as_nanos().try_into().unwrap_or(i64::MAX); - let timeout = dispatch_time(DISPATCH_TIME_NOW, nanos); - - let timeout = dispatch_semaphore_wait(self.semaphore, timeout) != 0; - - let state = self.state.swap(EMPTY, Acquire); - if state == NOTIFIED && timeout { - // If the state was NOTIFIED but semaphore_wait returned without - // decrementing the count because of a timeout, it means another - // thread is about to call semaphore_signal. We must wait for that - // to happen to ensure the semaphore count is reset. - while dispatch_semaphore_wait(self.semaphore, DISPATCH_TIME_FOREVER) != 0 {} - } else { - // Either a timeout occurred and we reset the state before any thread - // tried to wake us up, or we were woken up and reset the state, - // making sure to observe the state change with acquire ordering. - // Either way, the semaphore counter is now zero again. - } - } - - // Does not need `Pin`, but other implementation do. - pub fn unpark(self: Pin<&Self>) { - let state = self.state.swap(NOTIFIED, Release); - if state == PARKED { - unsafe { - dispatch_semaphore_signal(self.semaphore); - } - } - } -} - -impl Drop for Parker { - fn drop(&mut self) { - // SAFETY: - // We always ensure that the semaphore count is reset, so this will - // never cause an exception. - unsafe { - dispatch_release(self.semaphore); - } - } -} diff --git a/library/std/src/sys/sync/thread_parking/mod.rs b/library/std/src/sys/sync/thread_parking/mod.rs index f4d8fa0a58c11..4ad68c83cf8e3 100644 --- a/library/std/src/sys/sync/thread_parking/mod.rs +++ b/library/std/src/sys/sync/thread_parking/mod.rs @@ -2,6 +2,7 @@ cfg_if::cfg_if! { if #[cfg(any( all(target_os = "windows", not(target_vendor = "win7")), target_os = "linux", + target_vendor = "apple", target_os = "android", all(target_arch = "wasm32", target_feature = "atomics"), target_os = "freebsd",