Use futex-based synchronization on Apple platforms

library/std/src/sys/pal/unix/fd.rs

@@ -284,9 +284,9 @@ impl FileDesc {
         super::weak::weak!(fn preadv(libc::c_int, *const libc::iovec, libc::c_int, off64_t) -> isize);
 
         match preadv.get() {
-            Some(preadv) => {
+            Some(read) => {
                 let ret = cvt(unsafe {
-                    preadv(
+                    read(
                         self.as_raw_fd(),
                         bufs.as_mut_ptr() as *mut libc::iovec as *const libc::iovec,
                         cmp::min(bufs.len(), max_iov()) as libc::c_int,
@@ -477,9 +477,9 @@ impl FileDesc {
         super::weak::weak!(fn pwritev(libc::c_int, *const libc::iovec, libc::c_int, off64_t) -> isize);
 
         match pwritev.get() {
-            Some(pwritev) => {
+            Some(read) => {
                 let ret = cvt(unsafe {
-                    pwritev(
+                    read(
                         self.as_raw_fd(),
                         bufs.as_ptr() as *const libc::iovec,
                         cmp::min(bufs.len(), max_iov()) as libc::c_int,

library/std/src/sys/pal/unix/futex.rs

@@ -1,5 +1,6 @@
 #![cfg(any(
     target_os = "linux",
+    target_vendor = "apple",
     target_os = "android",
     all(target_os = "emscripten", target_feature = "atomics"),
     target_os = "freebsd",
@@ -145,6 +146,189 @@ pub fn futex_wake_all(futex: &AtomicU32) {
     };
 }
 
+/// With macOS version 14.4, Apple introduced a public futex API. Unfortunately,
+/// our minimum supported version is 10.12, so we need a fallback API. Luckily
+/// for us, the underlying syscalls have been available since exactly that
+/// version, so we just use those when needed. This is private API however,
+/// which means we need to take care to avoid breakage if the syscall is removed
+/// and to avoid apps being rejected from the App Store. To do this, we use weak
+/// linkage emulation for both the public and the private API.
+///
+/// See https://developer.apple.com/documentation/os/os_sync_wait_on_address?language=objc
+/// for documentation of the public API and
+/// https://github.com/apple-oss-distributions/xnu/blob/1031c584a5e37aff177559b9f69dbd3c8c3fd30a/bsd/sys/ulock.h#L69
+/// for the header file of the private API, along with its usage in libpthread
+/// https://github.com/apple-oss-distributions/libpthread/blob/d8c4e3c212553d3e0f5d76bb7d45a8acd61302dc/src/pthread_cond.c#L463
+#[cfg(target_vendor = "apple")]
+mod apple {
+    use crate::ffi::{c_int, c_void};
+    // Explicitly use `dlsym` for the private fallback API to ensure that the
+    // App Store checks do not flag Apps using Rust as relying on private API.
+    use crate::sys::pal::weak::{dlsym, weak};
+
+    pub const OS_CLOCK_MACH_ABSOLUTE_TIME: u32 = 32;
+    pub const OS_SYNC_WAIT_ON_ADDRESS_NONE: u32 = 0;
+    pub const OS_SYNC_WAKE_BY_ADDRESS_NONE: u32 = 0;
+
+    pub const UL_COMPARE_AND_WAIT: u32 = 1;
+    pub const ULF_WAKE_ALL: u32 = 0x100;
+    // The syscalls support directly returning errors instead of going through errno.
+    pub const ULF_NO_ERRNO: u32 = 0x1000000;
+
+    // These functions appeared with macOS 14.4, iOS 17.4, tvOS 17.4, watchOS 10.4, visionOS 1.1.
+    weak! {
+        pub fn os_sync_wait_on_address(*mut c_void, u64, usize, u32) -> c_int
+    }
+
+    weak! {
+        pub fn os_sync_wait_on_address_with_timeout(*mut c_void, u64, usize, u32, u32, u64) -> c_int
+    }
+
+    weak! {
+        pub fn os_sync_wake_by_address_any(*mut c_void, usize, u32) -> c_int
+    }
+
+    weak! {
+        pub fn os_sync_wake_by_address_all(*mut c_void, usize, u32) -> c_int
+    }
+
+    // This syscall appeared with macOS 11.0.
+    // It is used to support nanosecond precision for timeouts, among other features.
+    dlsym! {
+        pub fn __ulock_wait2(u32, *mut c_void, u64, u64, u64) -> c_int
+    }
+
+    // These syscalls appeared with macOS 10.12.
+    dlsym! {
+        pub fn __ulock_wait(u32, *mut c_void, u64, u32) -> c_int
+    }
+
+    dlsym! {
+        pub fn __ulock_wake(u32, *mut c_void, u64) -> c_int
+    }
+}
+
+#[cfg(target_vendor = "apple")]
+pub fn futex_wait(futex: &AtomicU32, expected: u32, timeout: Option<Duration>) -> bool {
+    use apple::*;
+
+    use crate::mem::size_of;
+
+    let addr = futex.as_ptr().cast();
+    let value = expected as u64;
+    let size = size_of::<u32>();
+    if let Some(timeout) = timeout {
+        let timeout_ns = timeout.as_nanos().clamp(1, u64::MAX as u128) as u64;
+        let timeout_ms = timeout.as_micros().clamp(1, u32::MAX as u128) as u32;
+
+        if let Some(wait) = os_sync_wait_on_address_with_timeout.get() {
+            let r = unsafe {
+                wait(
+                    addr,
+                    value,
+                    size,
+                    OS_SYNC_WAIT_ON_ADDRESS_NONE,
+                    OS_CLOCK_MACH_ABSOLUTE_TIME,
+                    timeout_ns,
+                )
+            };
+
+            // We promote spurious wakeups (reported as EINTR) to normal ones for
+            // simplicity.
+            r != -1 || super::os::errno() != libc::ETIMEDOUT
+        } else if let Some(wait) = __ulock_wait2.get() {
+            unsafe {
+                wait(UL_COMPARE_AND_WAIT | ULF_NO_ERRNO, addr, value, timeout_ns, 0)
+                    != -libc::ETIMEDOUT
+            }
+        } else if let Some(wait) = __ulock_wait.get() {
+            unsafe {
+                wait(UL_COMPARE_AND_WAIT | ULF_NO_ERRNO, addr, value, timeout_ms)
+                    != -libc::ETIMEDOUT
+            }
+        } else {
+            panic!("your system is below the minimum supported version of Rust");
+        }
+    } else {
+        if let Some(wait) = os_sync_wait_on_address.get() {
+            unsafe {
+                wait(addr, value, size, OS_SYNC_WAIT_ON_ADDRESS_NONE);
+            }
+        } else if let Some(wait) = __ulock_wait.get() {
+            unsafe {
+                wait(UL_COMPARE_AND_WAIT | ULF_NO_ERRNO, addr, value, 0);
+            }
+        } else {
+            panic!("your system is below the minimum supported version of Rust");
+        }
+        true
+    }
+}
+
+#[cfg(target_vendor = "apple")]
+pub fn futex_wake(futex: &AtomicU32) -> bool {
+    use apple::*;
+
+    use crate::io::Error;
+    use crate::mem::size_of;
+
+    let addr = futex.as_ptr().cast();
+    if let Some(wake) = os_sync_wake_by_address_any.get() {
+        unsafe { wake(addr, size_of::<u32>(), OS_SYNC_WAKE_BY_ADDRESS_NONE) == 0 }
+    } else if let Some(wake) = __ulock_wake.get() {
+        // Judging by its use in pthreads, __ulock_wake can get interrupted, so
+        // retry until either waking up a waiter or failing because there are no
+        // waiters (ENOENT).
+        loop {
+            let r = unsafe { wake(UL_COMPARE_AND_WAIT | ULF_NO_ERRNO, addr, 0) };
+
+            if r >= 0 {
+                return true;
+            } else {
+                match -r {
+                    libc::ENOENT => return false,
+                    libc::EINTR => continue,
+                    err => panic!("__ulock_wake failed: {}", Error::from_raw_os_error(err)),
+                }
+            }
+        }
+    } else {
+        panic!("your system is below the minimum supported version of Rust");
+    }
+}
+
+#[cfg(target_vendor = "apple")]
+pub fn futex_wake_all(futex: &AtomicU32) {
+    use apple::*;
+
+    use crate::io::Error;
+    use crate::mem::size_of;
+
+    let addr = futex.as_ptr().cast();
+
+    if let Some(wake) = os_sync_wake_by_address_all.get() {
+        unsafe {
+            wake(addr, size_of::<u32>(), OS_SYNC_WAKE_BY_ADDRESS_NONE);
+        }
+    } else if let Some(wake) = __ulock_wake.get() {
+        loop {
+            let r = unsafe { wake(UL_COMPARE_AND_WAIT | ULF_WAKE_ALL | ULF_NO_ERRNO, addr, 0) };
+
+            if r >= 0 {
+                return;
+            } else {
+                match -r {
+                    libc::ENOENT => return,
+                    libc::EINTR => continue,
+                    err => panic!("__ulock_wake failed: {}", Error::from_raw_os_error(err)),
+                }
+            }
+        }
+    } else {
+        panic!("your system is below the minimum supported version of Rust");
+    }
+}
+
 #[cfg(target_os = "openbsd")]
 pub fn futex_wait(futex: &AtomicU32, expected: u32, timeout: Option<Duration>) -> bool {
     use super::time::Timespec;

library/std/src/sys/pal/unix/sync/mod.rs

@@ -9,8 +9,10 @@
 )))]
 #![forbid(unsafe_op_in_unsafe_fn)]
 
+#[cfg(not(target_vendor = "apple"))]
 mod condvar;
 mod mutex;
 
+#[cfg(not(target_vendor = "apple"))]
 pub use condvar::Condvar;
 pub use mutex::Mutex;

library/std/src/sys/pal/unix/weak.rs

@@ -62,15 +62,16 @@ impl<F: Copy> ExternWeak<F> {
 }
 
 pub(crate) macro dlsym {
-    (fn $name:ident($($t:ty),*) -> $ret:ty) => (
-         dlsym!(fn $name($($t),*) -> $ret, stringify!($name));
+    ($v:vis fn $name:ident($($t:ty),*) -> $ret:ty) => (
+        dlsym!($v fn $name($($t),*) -> $ret, stringify!($name));
     ),
-    (fn $name:ident($($t:ty),*) -> $ret:ty, $sym:expr) => (
-        static DLSYM: DlsymWeak<unsafe extern "C" fn($($t),*) -> $ret> =
+    ($v:vis fn $name:ident($($t:ty),*) -> $ret:ty, $sym:expr) => (
+        #[allow(non_upper_case_globals)]
+        $v static $name: DlsymWeak<unsafe extern "C" fn($($t),*) -> $ret> =
             DlsymWeak::new(concat!($sym, '\0'));
-        let $name = &DLSYM;
     )
 }
+
 pub(crate) struct DlsymWeak<F> {
     name: &'static str,
     func: AtomicPtr<libc::c_void>,

library/std/src/sys/sync/condvar/mod.rs

@@ -2,6 +2,7 @@ cfg_if::cfg_if! {
     if #[cfg(any(
         all(target_os = "windows", not(target_vendor="win7")),
         target_os = "linux",
+        target_vendor = "apple",
         target_os = "android",
         target_os = "freebsd",
         target_os = "openbsd",

library/std/src/sys/sync/once/mod.rs

@@ -18,6 +18,10 @@ cfg_if::cfg_if! {
         target_os = "dragonfly",
         target_os = "fuchsia",
         target_os = "hermit",
+        target_os = "macos",
+        target_os = "ios",
+        target_os = "tvos",
+        target_os = "watchos",
     ))] {
         mod futex;
         pub use futex::{Once, OnceState};

library/std/src/sys/sync/rwlock/mod.rs

@@ -2,6 +2,7 @@ cfg_if::cfg_if! {
     if #[cfg(any(
         all(target_os = "windows", not(target_vendor = "win7")),
         target_os = "linux",
+        target_vendor = "apple",
         target_os = "android",
         target_os = "freebsd",
         target_os = "openbsd",