Auto merge of #2029 - RalfJung:simd-bitmask, r=RalfJung

implement simd bitmask intrinsics

Cc #1912

src/helpers.rs

@@ -775,18 +775,3 @@ pub fn isolation_abort_error(name: &str) -> InterpResult<'static> {
         name,
     )))
 }
-
-pub fn bool_to_simd_element(b: bool, size: Size) -> Scalar<Tag> {
-    // SIMD uses all-1 as pattern for "true"
-    let val = if b { -1 } else { 0 };
-    Scalar::from_int(val, size)
-}
-
-pub fn simd_element_to_bool<'tcx>(elem: ImmTy<'tcx, Tag>) -> InterpResult<'tcx, bool> {
-    let val = elem.to_scalar()?.to_int(elem.layout.size)?;
-    Ok(match val {
-        0 => false,
-        -1 => true,
-        _ => throw_ub_format!("each element of a SIMD mask must be all-0-bits or all-1-bits"),
-    })
-}

src/shims/intrinsics.rs

@@ -1,14 +1,15 @@
+use std::convert::TryInto;
 use std::iter;
 
 use log::trace;
 
 use rustc_apfloat::{Float, Round};
 use rustc_middle::ty::layout::{HasParamEnv, IntegerExt, LayoutOf};
 use rustc_middle::{mir, mir::BinOp, ty, ty::FloatTy};
-use rustc_target::abi::{Align, Integer};
+use rustc_target::abi::{Align, Endian, HasDataLayout, Integer, Size};
 
 use crate::*;
-use helpers::{bool_to_simd_element, check_arg_count, simd_element_to_bool};
+use helpers::check_arg_count;
 
 pub enum AtomicOp {
     MirOp(mir::BinOp, bool),
@@ -663,6 +664,45 @@ pub trait EvalContextExt<'mir, 'tcx: 'mir>: crate::MiriEvalContextExt<'mir, 'tcx
                     this.write_immediate(*val, &dest.into())?;
                 }
             }
+            "simd_select_bitmask" => {
+                let &[ref mask, ref yes, ref no] = check_arg_count(args)?;
+                let (yes, yes_len) = this.operand_to_simd(yes)?;
+                let (no, no_len) = this.operand_to_simd(no)?;
+                let (dest, dest_len) = this.place_to_simd(dest)?;
+                let bitmask_len = dest_len.max(8);
+
+                assert!(mask.layout.ty.is_integral());
+                assert!(bitmask_len <= 64);
+                assert_eq!(bitmask_len, mask.layout.size.bits());
+                assert_eq!(dest_len, yes_len);
+                assert_eq!(dest_len, no_len);
+
+                let mask: u64 = this
+                    .read_scalar(mask)?
+                    .check_init()?
+                    .to_bits(mask.layout.size)?
+                    .try_into()
+                    .unwrap();
+                for i in 0..dest_len {
+                    let mask =
+                        mask & (1 << simd_bitmask_index(i, dest_len, this.data_layout().endian));
+                    let yes = this.read_immediate(&this.mplace_index(&yes, i)?.into())?;
+                    let no = this.read_immediate(&this.mplace_index(&no, i)?.into())?;
+                    let dest = this.mplace_index(&dest, i)?;
+
+                    let val = if mask != 0 { yes } else { no };
+                    this.write_immediate(*val, &dest.into())?;
+                }
+                for i in dest_len..bitmask_len {
+                    // If the mask is "padded", ensure that padding is all-zero.
+                    let mask = mask & (1 << i);
+                    if mask != 0 {
+                        throw_ub_format!(
+                            "a SIMD bitmask less than 8 bits long must be filled with 0s for the remaining bits"
+                        );
+                    }
+                }
+            }
             #[rustfmt::skip]
             "simd_cast" | "simd_as" => {
                 let &[ref op] = check_arg_count(args)?;
@@ -787,6 +827,24 @@ pub trait EvalContextExt<'mir, 'tcx: 'mir>: crate::MiriEvalContextExt<'mir, 'tcx
                     }
                 }
             }
+            "simd_bitmask" => {
+                let &[ref op] = check_arg_count(args)?;
+                let (op, op_len) = this.operand_to_simd(op)?;
+                let bitmask_len = op_len.max(8);
+
+                assert!(dest.layout.ty.is_integral());
+                assert!(bitmask_len <= 64);
+                assert_eq!(bitmask_len, dest.layout.size.bits());
+
+                let mut res = 0u64;
+                for i in 0..op_len {
+                    let op = this.read_immediate(&this.mplace_index(&op, i)?.into())?;
+                    if simd_element_to_bool(op)? {
+                        res |= 1 << simd_bitmask_index(i, op_len, this.data_layout().endian);
+                    }
+                }
+                this.write_int(res, dest)?;
+            }
 
             // Atomic operations
             "atomic_load" => this.atomic_load(args, dest, AtomicReadOp::SeqCst)?,
@@ -1307,3 +1365,26 @@ fn fmin_op<'tcx>(
         FloatTy::F64 => Scalar::from_f64(left.to_f64()?.min(right.to_f64()?)),
     })
 }
+
+fn bool_to_simd_element(b: bool, size: Size) -> Scalar<Tag> {
+    // SIMD uses all-1 as pattern for "true"
+    let val = if b { -1 } else { 0 };
+    Scalar::from_int(val, size)
+}
+
+fn simd_element_to_bool<'tcx>(elem: ImmTy<'tcx, Tag>) -> InterpResult<'tcx, bool> {
+    let val = elem.to_scalar()?.to_int(elem.layout.size)?;
+    Ok(match val {
+        0 => false,
+        -1 => true,
+        _ => throw_ub_format!("each element of a SIMD mask must be all-0-bits or all-1-bits"),
+    })
+}
+
+fn simd_bitmask_index(idx: u64, vec_len: u64, endianess: Endian) -> u64 {
+    assert!(idx < vec_len);
+    match endianess {
+        Endian::Little => idx,
+        Endian::Big => vec_len - 1 - idx, // reverse order of bits
+    }
+}

tests/compile-fail/intrinsics/simd-select-bitmask-invalid.rs

@@ -0,0 +1,15 @@
+#![feature(platform_intrinsics, repr_simd)]
+
+extern "platform-intrinsic" {
+    fn simd_select_bitmask<M, T>(m: M, yes: T, no: T) -> T;
+}
+
+#[repr(simd)]
+#[allow(non_camel_case_types)]
+#[derive(Copy, Clone)]
+struct i32x2(i32, i32);
+
+fn main() { unsafe {
+    let x = i32x2(0, 1);
+    simd_select_bitmask(0b11111111u8, x, x); //~ERROR bitmask less than 8 bits long must be filled with 0s for the remaining bits
+} }

tests/compile-fail/intrinsics/simd-select-invalid-bool.rs

@@ -0,0 +1,15 @@
+#![feature(platform_intrinsics, repr_simd)]
+
+extern "platform-intrinsic" {
+    fn simd_select<M, T>(m: M, yes: T, no: T) -> T;
+}
+
+#[repr(simd)]
+#[allow(non_camel_case_types)]
+#[derive(Copy, Clone)]
+struct i32x2(i32, i32);
+
+fn main() { unsafe {
+    let x = i32x2(0, 1);
+    simd_select(x, x, x); //~ERROR must be all-0-bits or all-1-bits
+} }

tests/run-pass/portable-simd.rs

@@ -187,6 +187,21 @@ fn simd_mask() {
     let intmask = Mask::from_int(i32x4::from_array([0, -1, 0, 0]));
     assert_eq!(intmask, Mask::from_array([false, true, false, false]));
     assert_eq!(intmask.to_array(), [false, true, false, false]);
+
+    let values = [
+        true, false, false, true, false, false, true, false, true, true, false, false, false, true,
+        false, true,
+    ];
+    let mask = Mask::<i64, 16>::from_array(values);
+    let bitmask = mask.to_bitmask();
+    assert_eq!(bitmask, 0b1010001101001001);
+    assert_eq!(Mask::<i64, 16>::from_bitmask(bitmask), mask);
+
+    let values = [false, false, false, true];
+    let mask = Mask::<i64, 4>::from_array(values);
+    let bitmask = mask.to_bitmask();
+    // FIXME fails until https://github.com/rust-lang/portable-simd/pull/267 lands: assert_eq!(bitmask, 0b1000);
+    assert_eq!(Mask::<i64, 4>::from_bitmask(bitmask), mask);
 }
 
 fn simd_cast() {