@@ -3014,6 +3014,28 @@ macro_rules! iterator {
3014
3014
{ $( $mut_: tt ) * } ,
3015
3015
{ $( $extra: tt) * }
3016
3016
) => {
3017
+ // Returns the first element and moves the start of the iterator forwards by 1.
3018
+ // Greatly improves performance compared to an inlined function. The iterator
3019
+ // must not be empty.
3020
+ macro_rules! next_unchecked {
3021
+ ( $self: ident) => { & $( $mut_ ) * * $self. post_inc_start( 1 ) }
3022
+ }
3023
+
3024
+ // Returns the last element and moves the end of the iterator backwards by 1.
3025
+ // Greatly improves performance compared to an inlined function. The iterator
3026
+ // must not be empty.
3027
+ macro_rules! next_back_unchecked {
3028
+ ( $self: ident) => { & $( $mut_ ) * * $self. pre_dec_end( 1 ) }
3029
+ }
3030
+
3031
+ // Shrinks the iterator when T is a ZST, by moving the end of the iterator
3032
+ // backwards by `n`. `n` must not exceed `self.len()`.
3033
+ macro_rules! zst_shrink {
3034
+ ( $self: ident, $n: ident) => {
3035
+ $self. end = ( $self. end as * $raw_mut u8 ) . wrapping_offset( -$n) as * $raw_mut T ;
3036
+ }
3037
+ }
3038
+
3017
3039
impl <' a, T > $name<' a, T > {
3018
3040
// Helper function for creating a slice from the iterator.
3019
3041
#[ inline( always) ]
@@ -3023,12 +3045,11 @@ macro_rules! iterator {
3023
3045
3024
3046
// Helper function for moving the start of the iterator forwards by `offset` elements,
3025
3047
// returning the old start.
3026
- // Unsafe because the offset must be in-bounds or one-past-the-end .
3048
+ // Unsafe because the offset must not exceed `self.len()` .
3027
3049
#[ inline( always) ]
3028
3050
unsafe fn post_inc_start( & mut self , offset: isize ) -> * $raw_mut T {
3029
3051
if mem:: size_of:: <T >( ) == 0 {
3030
- // This is *reducing* the length. `ptr` never changes with ZST.
3031
- self . end = ( self . end as * $raw_mut u8 ) . wrapping_offset( -offset) as * $raw_mut T ;
3052
+ zst_shrink!( self , offset) ;
3032
3053
self . ptr
3033
3054
} else {
3034
3055
let old = self . ptr;
@@ -3039,11 +3060,11 @@ macro_rules! iterator {
3039
3060
3040
3061
// Helper function for moving the end of the iterator backwards by `offset` elements,
3041
3062
// returning the new end.
3042
- // Unsafe because the offset must be in-bounds or one-past-the-end .
3063
+ // Unsafe because the offset must not exceed `self.len()` .
3043
3064
#[ inline( always) ]
3044
3065
unsafe fn pre_dec_end( & mut self , offset: isize ) -> * $raw_mut T {
3045
3066
if mem:: size_of:: <T >( ) == 0 {
3046
- self . end = ( self . end as * $raw_mut u8 ) . wrapping_offset ( - offset) as * $raw_mut T ;
3067
+ zst_shrink! ( self , offset) ;
3047
3068
self . ptr
3048
3069
} else {
3049
3070
self . end = self . end. offset( -offset) ;
@@ -3080,7 +3101,7 @@ macro_rules! iterator {
3080
3101
if is_empty!( self ) {
3081
3102
None
3082
3103
} else {
3083
- Some ( & $ ( $mut_ ) * * self . post_inc_start ( 1 ) )
3104
+ Some ( next_unchecked! ( self ) )
3084
3105
}
3085
3106
}
3086
3107
}
@@ -3109,11 +3130,10 @@ macro_rules! iterator {
3109
3130
}
3110
3131
return None ;
3111
3132
}
3112
- // We are in bounds. `offset ` does the right thing even for ZSTs.
3133
+ // We are in bounds. `post_inc_start ` does the right thing even for ZSTs.
3113
3134
unsafe {
3114
- let elem = Some ( & $( $mut_ ) * * self . ptr. add( n) ) ;
3115
- self . post_inc_start( ( n as isize ) . wrapping_add( 1 ) ) ;
3116
- elem
3135
+ self . post_inc_start( n as isize ) ;
3136
+ Some ( next_unchecked!( self ) )
3117
3137
}
3118
3138
}
3119
3139
@@ -3130,13 +3150,13 @@ macro_rules! iterator {
3130
3150
let mut accum = init;
3131
3151
unsafe {
3132
3152
while len!( self ) >= 4 {
3133
- accum = f( accum, & $ ( $mut_ ) * * self . post_inc_start ( 1 ) ) ?;
3134
- accum = f( accum, & $ ( $mut_ ) * * self . post_inc_start ( 1 ) ) ?;
3135
- accum = f( accum, & $ ( $mut_ ) * * self . post_inc_start ( 1 ) ) ?;
3136
- accum = f( accum, & $ ( $mut_ ) * * self . post_inc_start ( 1 ) ) ?;
3153
+ accum = f( accum, next_unchecked! ( self ) ) ?;
3154
+ accum = f( accum, next_unchecked! ( self ) ) ?;
3155
+ accum = f( accum, next_unchecked! ( self ) ) ?;
3156
+ accum = f( accum, next_unchecked! ( self ) ) ?;
3137
3157
}
3138
3158
while !is_empty!( self ) {
3139
- accum = f( accum, & $ ( $mut_ ) * * self . post_inc_start ( 1 ) ) ?;
3159
+ accum = f( accum, next_unchecked! ( self ) ) ?;
3140
3160
}
3141
3161
}
3142
3162
Try :: from_ok( accum)
@@ -3207,11 +3227,25 @@ macro_rules! iterator {
3207
3227
if is_empty!( self ) {
3208
3228
None
3209
3229
} else {
3210
- Some ( & $ ( $mut_ ) * * self . pre_dec_end ( 1 ) )
3230
+ Some ( next_back_unchecked! ( self ) )
3211
3231
}
3212
3232
}
3213
3233
}
3214
3234
3235
+ #[ inline]
3236
+ fn nth_back( & mut self , n: usize ) -> Option <$elem> {
3237
+ if n >= len!( self ) {
3238
+ // This iterator is now empty.
3239
+ self . end = self . ptr;
3240
+ return None ;
3241
+ }
3242
+ // We are in bounds. `pre_dec_end` does the right thing even for ZSTs.
3243
+ unsafe {
3244
+ self . pre_dec_end( n as isize ) ;
3245
+ Some ( next_back_unchecked!( self ) )
3246
+ }
3247
+ }
3248
+
3215
3249
#[ inline]
3216
3250
fn try_rfold<B , F , R >( & mut self , init: B , mut f: F ) -> R where
3217
3251
Self : Sized , F : FnMut ( B , Self :: Item ) -> R , R : Try <Ok =B >
@@ -3220,14 +3254,14 @@ macro_rules! iterator {
3220
3254
let mut accum = init;
3221
3255
unsafe {
3222
3256
while len!( self ) >= 4 {
3223
- accum = f( accum, & $ ( $mut_ ) * * self . pre_dec_end ( 1 ) ) ?;
3224
- accum = f( accum, & $ ( $mut_ ) * * self . pre_dec_end ( 1 ) ) ?;
3225
- accum = f( accum, & $ ( $mut_ ) * * self . pre_dec_end ( 1 ) ) ?;
3226
- accum = f( accum, & $ ( $mut_ ) * * self . pre_dec_end ( 1 ) ) ?;
3257
+ accum = f( accum, next_back_unchecked! ( self ) ) ?;
3258
+ accum = f( accum, next_back_unchecked! ( self ) ) ?;
3259
+ accum = f( accum, next_back_unchecked! ( self ) ) ?;
3260
+ accum = f( accum, next_back_unchecked! ( self ) ) ?;
3227
3261
}
3228
3262
// inlining is_empty everywhere makes a huge performance difference
3229
3263
while !is_empty!( self ) {
3230
- accum = f( accum, & $ ( $mut_ ) * * self . pre_dec_end ( 1 ) ) ?;
3264
+ accum = f( accum, next_back_unchecked! ( self ) ) ?;
3231
3265
}
3232
3266
}
3233
3267
Try :: from_ok( accum)
0 commit comments