Skip to main content

core/stdarch/crates/core_arch/src/x86/
avx2.rs

1//! Advanced Vector Extensions 2 (AVX)
2//!
3//! AVX2 expands most AVX commands to 256-bit wide vector registers and
4//! adds [FMA](https://en.wikipedia.org/wiki/Fused_multiply-accumulate).
5//!
6//! The references are:
7//!
8//! - [Intel 64 and IA-32 Architectures Software Developer's Manual Volume 2:
9//!   Instruction Set Reference, A-Z][intel64_ref].
10//! - [AMD64 Architecture Programmer's Manual, Volume 3: General-Purpose and
11//!   System Instructions][amd64_ref].
12//!
13//! Wikipedia's [AVX][wiki_avx] and [FMA][wiki_fma] pages provide a quick
14//! overview of the instructions available.
15//!
16//! [intel64_ref]: https://www.intel.com/content/dam/www/public/us/en/documents/manuals/64-ia-32-architectures-software-developer-instruction-set-reference-manual-325383.pdf
17//! [amd64_ref]: https://docs.amd.com/v/u/en-US/24594_3.37
18//! [wiki_avx]: https://en.wikipedia.org/wiki/Advanced_Vector_Extensions
19//! [wiki_fma]: https://en.wikipedia.org/wiki/Fused_multiply-accumulate
20
21use crate::core_arch::{simd::*, x86::*};
22use crate::intrinsics::simd::*;
23
24#[cfg(test)]
25use stdarch_test::assert_instr;
26
27/// Computes the absolute values of packed 32-bit integers in `a`.
28///
29/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm256_abs_epi32)
30#[inline]
31#[target_feature(enable = "avx2")]
32#[cfg_attr(test, assert_instr(vpabsd))]
33#[stable(feature = "simd_x86", since = "1.27.0")]
34#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")]
35pub const fn _mm256_abs_epi32(a: __m256i) -> __m256i {
36    unsafe {
37        let a = a.as_i32x8();
38        let r = simd_select::<m32x8, _>(simd_lt(a, i32x8::ZERO), simd_neg(a), a);
39        transmute(r)
40    }
41}
42
43/// Computes the absolute values of packed 16-bit integers in `a`.
44///
45/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm256_abs_epi16)
46#[inline]
47#[target_feature(enable = "avx2")]
48#[cfg_attr(test, assert_instr(vpabsw))]
49#[stable(feature = "simd_x86", since = "1.27.0")]
50#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")]
51pub const fn _mm256_abs_epi16(a: __m256i) -> __m256i {
52    unsafe {
53        let a = a.as_i16x16();
54        let r = simd_select::<m16x16, _>(simd_lt(a, i16x16::ZERO), simd_neg(a), a);
55        transmute(r)
56    }
57}
58
59/// Computes the absolute values of packed 8-bit integers in `a`.
60///
61/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm256_abs_epi8)
62#[inline]
63#[target_feature(enable = "avx2")]
64#[cfg_attr(test, assert_instr(vpabsb))]
65#[stable(feature = "simd_x86", since = "1.27.0")]
66#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")]
67pub const fn _mm256_abs_epi8(a: __m256i) -> __m256i {
68    unsafe {
69        let a = a.as_i8x32();
70        let r = simd_select::<m8x32, _>(simd_lt(a, i8x32::ZERO), simd_neg(a), a);
71        transmute(r)
72    }
73}
74
75/// Adds packed 64-bit integers in `a` and `b`.
76///
77/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm256_add_epi64)
78#[inline]
79#[target_feature(enable = "avx2")]
80#[cfg_attr(test, assert_instr(vpaddq))]
81#[stable(feature = "simd_x86", since = "1.27.0")]
82#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")]
83pub const fn _mm256_add_epi64(a: __m256i, b: __m256i) -> __m256i {
84    unsafe { transmute(simd_add(a.as_i64x4(), b.as_i64x4())) }
85}
86
87/// Adds packed 32-bit integers in `a` and `b`.
88///
89/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm256_add_epi32)
90#[inline]
91#[target_feature(enable = "avx2")]
92#[cfg_attr(test, assert_instr(vpaddd))]
93#[stable(feature = "simd_x86", since = "1.27.0")]
94#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")]
95pub const fn _mm256_add_epi32(a: __m256i, b: __m256i) -> __m256i {
96    unsafe { transmute(simd_add(a.as_i32x8(), b.as_i32x8())) }
97}
98
99/// Adds packed 16-bit integers in `a` and `b`.
100///
101/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm256_add_epi16)
102#[inline]
103#[target_feature(enable = "avx2")]
104#[cfg_attr(test, assert_instr(vpaddw))]
105#[stable(feature = "simd_x86", since = "1.27.0")]
106#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")]
107pub const fn _mm256_add_epi16(a: __m256i, b: __m256i) -> __m256i {
108    unsafe { transmute(simd_add(a.as_i16x16(), b.as_i16x16())) }
109}
110
111/// Adds packed 8-bit integers in `a` and `b`.
112///
113/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm256_add_epi8)
114#[inline]
115#[target_feature(enable = "avx2")]
116#[cfg_attr(test, assert_instr(vpaddb))]
117#[stable(feature = "simd_x86", since = "1.27.0")]
118#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")]
119pub const fn _mm256_add_epi8(a: __m256i, b: __m256i) -> __m256i {
120    unsafe { transmute(simd_add(a.as_i8x32(), b.as_i8x32())) }
121}
122
123/// Adds packed 8-bit integers in `a` and `b` using saturation.
124///
125/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm256_adds_epi8)
126#[inline]
127#[target_feature(enable = "avx2")]
128#[cfg_attr(test, assert_instr(vpaddsb))]
129#[stable(feature = "simd_x86", since = "1.27.0")]
130#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")]
131pub const fn _mm256_adds_epi8(a: __m256i, b: __m256i) -> __m256i {
132    unsafe { transmute(simd_saturating_add(a.as_i8x32(), b.as_i8x32())) }
133}
134
135/// Adds packed 16-bit integers in `a` and `b` using saturation.
136///
137/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm256_adds_epi16)
138#[inline]
139#[target_feature(enable = "avx2")]
140#[cfg_attr(test, assert_instr(vpaddsw))]
141#[stable(feature = "simd_x86", since = "1.27.0")]
142#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")]
143pub const fn _mm256_adds_epi16(a: __m256i, b: __m256i) -> __m256i {
144    unsafe { transmute(simd_saturating_add(a.as_i16x16(), b.as_i16x16())) }
145}
146
147/// Adds packed unsigned 8-bit integers in `a` and `b` using saturation.
148///
149/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm256_adds_epu8)
150#[inline]
151#[target_feature(enable = "avx2")]
152#[cfg_attr(test, assert_instr(vpaddusb))]
153#[stable(feature = "simd_x86", since = "1.27.0")]
154#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")]
155pub const fn _mm256_adds_epu8(a: __m256i, b: __m256i) -> __m256i {
156    unsafe { transmute(simd_saturating_add(a.as_u8x32(), b.as_u8x32())) }
157}
158
159/// Adds packed unsigned 16-bit integers in `a` and `b` using saturation.
160///
161/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm256_adds_epu16)
162#[inline]
163#[target_feature(enable = "avx2")]
164#[cfg_attr(test, assert_instr(vpaddusw))]
165#[stable(feature = "simd_x86", since = "1.27.0")]
166#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")]
167pub const fn _mm256_adds_epu16(a: __m256i, b: __m256i) -> __m256i {
168    unsafe { transmute(simd_saturating_add(a.as_u16x16(), b.as_u16x16())) }
169}
170
171/// Concatenates pairs of 16-byte blocks in `a` and `b` into a 32-byte temporary
172/// result, shifts the result right by `n` bytes, and returns the low 16 bytes.
173///
174/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm256_alignr_epi8)
175#[inline]
176#[target_feature(enable = "avx2")]
177#[cfg_attr(test, assert_instr(vpalignr, IMM8 = 7))]
178#[rustc_legacy_const_generics(2)]
179#[stable(feature = "simd_x86", since = "1.27.0")]
180#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")]
181pub const fn _mm256_alignr_epi8<const IMM8: i32>(a: __m256i, b: __m256i) -> __m256i {
182    static_assert_uimm_bits!(IMM8, 8);
183
184    // If palignr is shifting the pair of vectors more than the size of two
185    // lanes, emit zero.
186    if IMM8 >= 32 {
187        return _mm256_setzero_si256();
188    }
189    // If palignr is shifting the pair of input vectors more than one lane,
190    // but less than two lanes, convert to shifting in zeroes.
191    let (a, b) = if IMM8 > 16 {
192        (_mm256_setzero_si256(), a)
193    } else {
194        (a, b)
195    };
196    unsafe {
197        if IMM8 == 16 {
198            return transmute(a);
199        }
200    }
201    const fn mask(shift: u32, i: u32) -> u32 {
202        let shift = shift % 16;
203        let mod_i = i % 16;
204        if mod_i < (16 - shift) {
205            i + shift
206        } else {
207            i + 16 + shift
208        }
209    }
210
211    unsafe {
212        let r: i8x32 = simd_shuffle!(
213            b.as_i8x32(),
214            a.as_i8x32(),
215            [
216                mask(IMM8 as u32, 0),
217                mask(IMM8 as u32, 1),
218                mask(IMM8 as u32, 2),
219                mask(IMM8 as u32, 3),
220                mask(IMM8 as u32, 4),
221                mask(IMM8 as u32, 5),
222                mask(IMM8 as u32, 6),
223                mask(IMM8 as u32, 7),
224                mask(IMM8 as u32, 8),
225                mask(IMM8 as u32, 9),
226                mask(IMM8 as u32, 10),
227                mask(IMM8 as u32, 11),
228                mask(IMM8 as u32, 12),
229                mask(IMM8 as u32, 13),
230                mask(IMM8 as u32, 14),
231                mask(IMM8 as u32, 15),
232                mask(IMM8 as u32, 16),
233                mask(IMM8 as u32, 17),
234                mask(IMM8 as u32, 18),
235                mask(IMM8 as u32, 19),
236                mask(IMM8 as u32, 20),
237                mask(IMM8 as u32, 21),
238                mask(IMM8 as u32, 22),
239                mask(IMM8 as u32, 23),
240                mask(IMM8 as u32, 24),
241                mask(IMM8 as u32, 25),
242                mask(IMM8 as u32, 26),
243                mask(IMM8 as u32, 27),
244                mask(IMM8 as u32, 28),
245                mask(IMM8 as u32, 29),
246                mask(IMM8 as u32, 30),
247                mask(IMM8 as u32, 31),
248            ],
249        );
250        transmute(r)
251    }
252}
253
254/// Computes the bitwise AND of 256 bits (representing integer data)
255/// in `a` and `b`.
256///
257/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm256_and_si256)
258#[inline]
259#[target_feature(enable = "avx2")]
260#[cfg_attr(test, assert_instr(vandps))]
261#[stable(feature = "simd_x86", since = "1.27.0")]
262#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")]
263pub const fn _mm256_and_si256(a: __m256i, b: __m256i) -> __m256i {
264    unsafe { transmute(simd_and(a.as_i64x4(), b.as_i64x4())) }
265}
266
267/// Computes the bitwise NOT of 256 bits (representing integer data)
268/// in `a` and then AND with `b`.
269///
270/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm256_andnot_si256)
271#[inline]
272#[target_feature(enable = "avx2")]
273#[cfg_attr(test, assert_instr(vandnps))]
274#[stable(feature = "simd_x86", since = "1.27.0")]
275#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")]
276pub const fn _mm256_andnot_si256(a: __m256i, b: __m256i) -> __m256i {
277    unsafe {
278        let all_ones = _mm256_set1_epi8(-1);
279        transmute(simd_and(
280            simd_xor(a.as_i64x4(), all_ones.as_i64x4()),
281            b.as_i64x4(),
282        ))
283    }
284}
285
286/// Averages packed unsigned 16-bit integers in `a` and `b`.
287///
288/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm256_avg_epu16)
289#[inline]
290#[target_feature(enable = "avx2")]
291#[cfg_attr(test, assert_instr(vpavgw))]
292#[stable(feature = "simd_x86", since = "1.27.0")]
293pub fn _mm256_avg_epu16(a: __m256i, b: __m256i) -> __m256i {
294    unsafe { transmute(pavgw(a.as_u16x16(), b.as_u16x16())) }
295}
296
297/// Averages packed unsigned 8-bit integers in `a` and `b`.
298///
299/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm256_avg_epu8)
300#[inline]
301#[target_feature(enable = "avx2")]
302#[cfg_attr(test, assert_instr(vpavgb))]
303#[stable(feature = "simd_x86", since = "1.27.0")]
304pub fn _mm256_avg_epu8(a: __m256i, b: __m256i) -> __m256i {
305    unsafe { transmute(pavgb(a.as_u8x32(), b.as_u8x32())) }
306}
307
308/// Blends packed 32-bit integers from `a` and `b` using control mask `IMM4`.
309///
310/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_blend_epi32)
311#[inline]
312#[target_feature(enable = "avx2")]
313#[cfg_attr(test, assert_instr(vblendps, IMM4 = 9))]
314#[rustc_legacy_const_generics(2)]
315#[stable(feature = "simd_x86", since = "1.27.0")]
316#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")]
317pub const fn _mm_blend_epi32<const IMM4: i32>(a: __m128i, b: __m128i) -> __m128i {
318    static_assert_uimm_bits!(IMM4, 4);
319    unsafe {
320        let a = a.as_i32x4();
321        let b = b.as_i32x4();
322        let r: i32x4 = simd_shuffle!(
323            a,
324            b,
325            [
326                [0, 4, 0, 4][IMM4 as usize & 0b11],
327                [1, 1, 5, 5][IMM4 as usize & 0b11],
328                [2, 6, 2, 6][(IMM4 as usize >> 2) & 0b11],
329                [3, 3, 7, 7][(IMM4 as usize >> 2) & 0b11],
330            ],
331        );
332        transmute(r)
333    }
334}
335
336/// Blends packed 32-bit integers from `a` and `b` using control mask `IMM8`.
337///
338/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm256_blend_epi32)
339#[inline]
340#[target_feature(enable = "avx2")]
341#[cfg_attr(test, assert_instr(vblendps, IMM8 = 9))]
342#[rustc_legacy_const_generics(2)]
343#[stable(feature = "simd_x86", since = "1.27.0")]
344#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")]
345pub const fn _mm256_blend_epi32<const IMM8: i32>(a: __m256i, b: __m256i) -> __m256i {
346    static_assert_uimm_bits!(IMM8, 8);
347    unsafe {
348        let a = a.as_i32x8();
349        let b = b.as_i32x8();
350        let r: i32x8 = simd_shuffle!(
351            a,
352            b,
353            [
354                [0, 8, 0, 8][IMM8 as usize & 0b11],
355                [1, 1, 9, 9][IMM8 as usize & 0b11],
356                [2, 10, 2, 10][(IMM8 as usize >> 2) & 0b11],
357                [3, 3, 11, 11][(IMM8 as usize >> 2) & 0b11],
358                [4, 12, 4, 12][(IMM8 as usize >> 4) & 0b11],
359                [5, 5, 13, 13][(IMM8 as usize >> 4) & 0b11],
360                [6, 14, 6, 14][(IMM8 as usize >> 6) & 0b11],
361                [7, 7, 15, 15][(IMM8 as usize >> 6) & 0b11],
362            ],
363        );
364        transmute(r)
365    }
366}
367
368/// Blends packed 16-bit integers from `a` and `b` using control mask `IMM8`.
369///
370/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm256_blend_epi16)
371#[inline]
372#[target_feature(enable = "avx2")]
373#[cfg_attr(test, assert_instr(vpblendw, IMM8 = 9))]
374#[rustc_legacy_const_generics(2)]
375#[stable(feature = "simd_x86", since = "1.27.0")]
376#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")]
377pub const fn _mm256_blend_epi16<const IMM8: i32>(a: __m256i, b: __m256i) -> __m256i {
378    static_assert_uimm_bits!(IMM8, 8);
379    unsafe {
380        let a = a.as_i16x16();
381        let b = b.as_i16x16();
382
383        let r: i16x16 = simd_shuffle!(
384            a,
385            b,
386            [
387                [0, 16, 0, 16][IMM8 as usize & 0b11],
388                [1, 1, 17, 17][IMM8 as usize & 0b11],
389                [2, 18, 2, 18][(IMM8 as usize >> 2) & 0b11],
390                [3, 3, 19, 19][(IMM8 as usize >> 2) & 0b11],
391                [4, 20, 4, 20][(IMM8 as usize >> 4) & 0b11],
392                [5, 5, 21, 21][(IMM8 as usize >> 4) & 0b11],
393                [6, 22, 6, 22][(IMM8 as usize >> 6) & 0b11],
394                [7, 7, 23, 23][(IMM8 as usize >> 6) & 0b11],
395                [8, 24, 8, 24][IMM8 as usize & 0b11],
396                [9, 9, 25, 25][IMM8 as usize & 0b11],
397                [10, 26, 10, 26][(IMM8 as usize >> 2) & 0b11],
398                [11, 11, 27, 27][(IMM8 as usize >> 2) & 0b11],
399                [12, 28, 12, 28][(IMM8 as usize >> 4) & 0b11],
400                [13, 13, 29, 29][(IMM8 as usize >> 4) & 0b11],
401                [14, 30, 14, 30][(IMM8 as usize >> 6) & 0b11],
402                [15, 15, 31, 31][(IMM8 as usize >> 6) & 0b11],
403            ],
404        );
405        transmute(r)
406    }
407}
408
409/// Blends packed 8-bit integers from `a` and `b` using `mask`.
410///
411/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm256_blendv_epi8)
412#[inline]
413#[target_feature(enable = "avx2")]
414#[cfg_attr(test, assert_instr(vpblendvb))]
415#[stable(feature = "simd_x86", since = "1.27.0")]
416#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")]
417pub const fn _mm256_blendv_epi8(a: __m256i, b: __m256i, mask: __m256i) -> __m256i {
418    unsafe {
419        let mask: i8x32 = simd_lt(mask.as_i8x32(), i8x32::ZERO);
420        transmute(simd_select(mask, b.as_i8x32(), a.as_i8x32()))
421    }
422}
423
424/// Broadcasts the low packed 8-bit integer from `a` to all elements of
425/// the 128-bit returned value.
426///
427/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_broadcastb_epi8)
428#[inline]
429#[target_feature(enable = "avx2")]
430#[cfg_attr(test, assert_instr(vpbroadcastb))]
431#[stable(feature = "simd_x86", since = "1.27.0")]
432#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")]
433pub const fn _mm_broadcastb_epi8(a: __m128i) -> __m128i {
434    unsafe {
435        let ret = simd_shuffle!(a.as_i8x16(), i8x16::ZERO, [0_u32; 16]);
436        transmute::<i8x16, _>(ret)
437    }
438}
439
440/// Broadcasts the low packed 8-bit integer from `a` to all elements of
441/// the 256-bit returned value.
442///
443/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm256_broadcastb_epi8)
444#[inline]
445#[target_feature(enable = "avx2")]
446#[cfg_attr(test, assert_instr(vpbroadcastb))]
447#[stable(feature = "simd_x86", since = "1.27.0")]
448#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")]
449pub const fn _mm256_broadcastb_epi8(a: __m128i) -> __m256i {
450    unsafe {
451        let ret = simd_shuffle!(a.as_i8x16(), i8x16::ZERO, [0_u32; 32]);
452        transmute::<i8x32, _>(ret)
453    }
454}
455
456// N.B., `simd_shuffle4` with integer data types for `a` and `b` is
457// often compiled to `vbroadcastss`.
458/// Broadcasts the low packed 32-bit integer from `a` to all elements of
459/// the 128-bit returned value.
460///
461/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_broadcastd_epi32)
462#[inline]
463#[target_feature(enable = "avx2")]
464#[cfg_attr(test, assert_instr(vbroadcastss))]
465#[stable(feature = "simd_x86", since = "1.27.0")]
466#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")]
467pub const fn _mm_broadcastd_epi32(a: __m128i) -> __m128i {
468    unsafe {
469        let ret = simd_shuffle!(a.as_i32x4(), i32x4::ZERO, [0_u32; 4]);
470        transmute::<i32x4, _>(ret)
471    }
472}
473
474// N.B., `simd_shuffle4`` with integer data types for `a` and `b` is
475// often compiled to `vbroadcastss`.
476/// Broadcasts the low packed 32-bit integer from `a` to all elements of
477/// the 256-bit returned value.
478///
479/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm256_broadcastd_epi32)
480#[inline]
481#[target_feature(enable = "avx2")]
482#[cfg_attr(test, assert_instr(vbroadcastss))]
483#[stable(feature = "simd_x86", since = "1.27.0")]
484#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")]
485pub const fn _mm256_broadcastd_epi32(a: __m128i) -> __m256i {
486    unsafe {
487        let ret = simd_shuffle!(a.as_i32x4(), i32x4::ZERO, [0_u32; 8]);
488        transmute::<i32x8, _>(ret)
489    }
490}
491
492/// Broadcasts the low packed 64-bit integer from `a` to all elements of
493/// the 128-bit returned value.
494///
495/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_broadcastq_epi64)
496#[inline]
497#[target_feature(enable = "avx2")]
498// Emits `vmovddup` instead of `vpbroadcastq`
499// See https://github.com/rust-lang/stdarch/issues/791
500#[cfg_attr(test, assert_instr(vmovddup))]
501#[stable(feature = "simd_x86", since = "1.27.0")]
502#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")]
503pub const fn _mm_broadcastq_epi64(a: __m128i) -> __m128i {
504    unsafe {
505        let ret = simd_shuffle!(a.as_i64x2(), a.as_i64x2(), [0_u32; 2]);
506        transmute::<i64x2, _>(ret)
507    }
508}
509
510/// Broadcasts the low packed 64-bit integer from `a` to all elements of
511/// the 256-bit returned value.
512///
513/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm256_broadcastq_epi64)
514#[inline]
515#[target_feature(enable = "avx2")]
516#[cfg_attr(test, assert_instr(vbroadcastsd))]
517#[stable(feature = "simd_x86", since = "1.27.0")]
518#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")]
519pub const fn _mm256_broadcastq_epi64(a: __m128i) -> __m256i {
520    unsafe {
521        let ret = simd_shuffle!(a.as_i64x2(), a.as_i64x2(), [0_u32; 4]);
522        transmute::<i64x4, _>(ret)
523    }
524}
525
526/// Broadcasts the low double-precision (64-bit) floating-point element
527/// from `a` to all elements of the 128-bit returned value.
528///
529/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_broadcastsd_pd)
530#[inline]
531#[target_feature(enable = "avx2")]
532#[cfg_attr(test, assert_instr(vmovddup))]
533#[stable(feature = "simd_x86", since = "1.27.0")]
534#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")]
535pub const fn _mm_broadcastsd_pd(a: __m128d) -> __m128d {
536    unsafe { simd_shuffle!(a, _mm_setzero_pd(), [0_u32; 2]) }
537}
538
539/// Broadcasts the low double-precision (64-bit) floating-point element
540/// from `a` to all elements of the 256-bit returned value.
541///
542/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm256_broadcastsd_pd)
543#[inline]
544#[target_feature(enable = "avx2")]
545#[cfg_attr(test, assert_instr(vbroadcastsd))]
546#[stable(feature = "simd_x86", since = "1.27.0")]
547#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")]
548pub const fn _mm256_broadcastsd_pd(a: __m128d) -> __m256d {
549    unsafe { simd_shuffle!(a, _mm_setzero_pd(), [0_u32; 4]) }
550}
551
552/// Broadcasts 128 bits of integer data from a to all 128-bit lanes in
553/// the 256-bit returned value.
554///
555/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_broadcastsi128_si256)
556#[inline]
557#[target_feature(enable = "avx2")]
558#[stable(feature = "simd_x86_updates", since = "1.82.0")]
559#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")]
560pub const fn _mm_broadcastsi128_si256(a: __m128i) -> __m256i {
561    unsafe {
562        let ret = simd_shuffle!(a.as_i64x2(), i64x2::ZERO, [0, 1, 0, 1]);
563        transmute::<i64x4, _>(ret)
564    }
565}
566
567// N.B., `broadcastsi128_si256` is often compiled to `vinsertf128` or
568// `vbroadcastf128`.
569/// Broadcasts 128 bits of integer data from a to all 128-bit lanes in
570/// the 256-bit returned value.
571///
572/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm256_broadcastsi128_si256)
573#[inline]
574#[target_feature(enable = "avx2")]
575#[stable(feature = "simd_x86", since = "1.27.0")]
576#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")]
577pub const fn _mm256_broadcastsi128_si256(a: __m128i) -> __m256i {
578    unsafe {
579        let ret = simd_shuffle!(a.as_i64x2(), i64x2::ZERO, [0, 1, 0, 1]);
580        transmute::<i64x4, _>(ret)
581    }
582}
583
584/// Broadcasts the low single-precision (32-bit) floating-point element
585/// from `a` to all elements of the 128-bit returned value.
586///
587/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_broadcastss_ps)
588#[inline]
589#[target_feature(enable = "avx2")]
590#[cfg_attr(test, assert_instr(vbroadcastss))]
591#[stable(feature = "simd_x86", since = "1.27.0")]
592#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")]
593pub const fn _mm_broadcastss_ps(a: __m128) -> __m128 {
594    unsafe { simd_shuffle!(a, _mm_setzero_ps(), [0_u32; 4]) }
595}
596
597/// Broadcasts the low single-precision (32-bit) floating-point element
598/// from `a` to all elements of the 256-bit returned value.
599///
600/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm256_broadcastss_ps)
601#[inline]
602#[target_feature(enable = "avx2")]
603#[cfg_attr(test, assert_instr(vbroadcastss))]
604#[stable(feature = "simd_x86", since = "1.27.0")]
605#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")]
606pub const fn _mm256_broadcastss_ps(a: __m128) -> __m256 {
607    unsafe { simd_shuffle!(a, _mm_setzero_ps(), [0_u32; 8]) }
608}
609
610/// Broadcasts the low packed 16-bit integer from a to all elements of
611/// the 128-bit returned value
612///
613/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_broadcastw_epi16)
614#[inline]
615#[target_feature(enable = "avx2")]
616#[cfg_attr(test, assert_instr(vpbroadcastw))]
617#[stable(feature = "simd_x86", since = "1.27.0")]
618#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")]
619pub const fn _mm_broadcastw_epi16(a: __m128i) -> __m128i {
620    unsafe {
621        let ret = simd_shuffle!(a.as_i16x8(), i16x8::ZERO, [0_u32; 8]);
622        transmute::<i16x8, _>(ret)
623    }
624}
625
626/// Broadcasts the low packed 16-bit integer from a to all elements of
627/// the 256-bit returned value
628///
629/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm256_broadcastw_epi16)
630#[inline]
631#[target_feature(enable = "avx2")]
632#[cfg_attr(test, assert_instr(vpbroadcastw))]
633#[stable(feature = "simd_x86", since = "1.27.0")]
634#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")]
635pub const fn _mm256_broadcastw_epi16(a: __m128i) -> __m256i {
636    unsafe {
637        let ret = simd_shuffle!(a.as_i16x8(), i16x8::ZERO, [0_u32; 16]);
638        transmute::<i16x16, _>(ret)
639    }
640}
641
642/// Compares packed 64-bit integers in `a` and `b` for equality.
643///
644/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm256_cmpeq_epi64)
645#[inline]
646#[target_feature(enable = "avx2")]
647#[cfg_attr(test, assert_instr(vpcmpeqq))]
648#[stable(feature = "simd_x86", since = "1.27.0")]
649#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")]
650pub const fn _mm256_cmpeq_epi64(a: __m256i, b: __m256i) -> __m256i {
651    unsafe { transmute::<i64x4, _>(simd_eq(a.as_i64x4(), b.as_i64x4())) }
652}
653
654/// Compares packed 32-bit integers in `a` and `b` for equality.
655///
656/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm256_cmpeq_epi32)
657#[inline]
658#[target_feature(enable = "avx2")]
659#[cfg_attr(test, assert_instr(vpcmpeqd))]
660#[stable(feature = "simd_x86", since = "1.27.0")]
661#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")]
662pub const fn _mm256_cmpeq_epi32(a: __m256i, b: __m256i) -> __m256i {
663    unsafe { transmute::<i32x8, _>(simd_eq(a.as_i32x8(), b.as_i32x8())) }
664}
665
666/// Compares packed 16-bit integers in `a` and `b` for equality.
667///
668/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm256_cmpeq_epi16)
669#[inline]
670#[target_feature(enable = "avx2")]
671#[cfg_attr(test, assert_instr(vpcmpeqw))]
672#[stable(feature = "simd_x86", since = "1.27.0")]
673#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")]
674pub const fn _mm256_cmpeq_epi16(a: __m256i, b: __m256i) -> __m256i {
675    unsafe { transmute::<i16x16, _>(simd_eq(a.as_i16x16(), b.as_i16x16())) }
676}
677
678/// Compares packed 8-bit integers in `a` and `b` for equality.
679///
680/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm256_cmpeq_epi8)
681#[inline]
682#[target_feature(enable = "avx2")]
683#[cfg_attr(test, assert_instr(vpcmpeqb))]
684#[stable(feature = "simd_x86", since = "1.27.0")]
685#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")]
686pub const fn _mm256_cmpeq_epi8(a: __m256i, b: __m256i) -> __m256i {
687    unsafe { transmute::<i8x32, _>(simd_eq(a.as_i8x32(), b.as_i8x32())) }
688}
689
690/// Compares packed 64-bit integers in `a` and `b` for greater-than.
691///
692/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm256_cmpgt_epi64)
693#[inline]
694#[target_feature(enable = "avx2")]
695#[cfg_attr(test, assert_instr(vpcmpgtq))]
696#[stable(feature = "simd_x86", since = "1.27.0")]
697#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")]
698pub const fn _mm256_cmpgt_epi64(a: __m256i, b: __m256i) -> __m256i {
699    unsafe { transmute::<i64x4, _>(simd_gt(a.as_i64x4(), b.as_i64x4())) }
700}
701
702/// Compares packed 32-bit integers in `a` and `b` for greater-than.
703///
704/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm256_cmpgt_epi32)
705#[inline]
706#[target_feature(enable = "avx2")]
707#[cfg_attr(test, assert_instr(vpcmpgtd))]
708#[stable(feature = "simd_x86", since = "1.27.0")]
709#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")]
710pub const fn _mm256_cmpgt_epi32(a: __m256i, b: __m256i) -> __m256i {
711    unsafe { transmute::<i32x8, _>(simd_gt(a.as_i32x8(), b.as_i32x8())) }
712}
713
714/// Compares packed 16-bit integers in `a` and `b` for greater-than.
715///
716/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm256_cmpgt_epi16)
717#[inline]
718#[target_feature(enable = "avx2")]
719#[cfg_attr(test, assert_instr(vpcmpgtw))]
720#[stable(feature = "simd_x86", since = "1.27.0")]
721#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")]
722pub const fn _mm256_cmpgt_epi16(a: __m256i, b: __m256i) -> __m256i {
723    unsafe { transmute::<i16x16, _>(simd_gt(a.as_i16x16(), b.as_i16x16())) }
724}
725
726/// Compares packed 8-bit integers in `a` and `b` for greater-than.
727///
728/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm256_cmpgt_epi8)
729#[inline]
730#[target_feature(enable = "avx2")]
731#[cfg_attr(test, assert_instr(vpcmpgtb))]
732#[stable(feature = "simd_x86", since = "1.27.0")]
733#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")]
734pub const fn _mm256_cmpgt_epi8(a: __m256i, b: __m256i) -> __m256i {
735    unsafe { transmute::<i8x32, _>(simd_gt(a.as_i8x32(), b.as_i8x32())) }
736}
737
738/// Sign-extend 16-bit integers to 32-bit integers.
739///
740/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm256_cvtepi16_epi32)
741#[inline]
742#[target_feature(enable = "avx2")]
743#[cfg_attr(test, assert_instr(vpmovsxwd))]
744#[stable(feature = "simd_x86", since = "1.27.0")]
745#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")]
746pub const fn _mm256_cvtepi16_epi32(a: __m128i) -> __m256i {
747    unsafe { transmute::<i32x8, _>(simd_cast(a.as_i16x8())) }
748}
749
750/// Sign-extend 16-bit integers to 64-bit integers.
751///
752/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm256_cvtepi16_epi64)
753#[inline]
754#[target_feature(enable = "avx2")]
755#[cfg_attr(test, assert_instr(vpmovsxwq))]
756#[stable(feature = "simd_x86", since = "1.27.0")]
757#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")]
758pub const fn _mm256_cvtepi16_epi64(a: __m128i) -> __m256i {
759    unsafe {
760        let a = a.as_i16x8();
761        let v64: i16x4 = simd_shuffle!(a, a, [0, 1, 2, 3]);
762        transmute::<i64x4, _>(simd_cast(v64))
763    }
764}
765
766/// Sign-extend 32-bit integers to 64-bit integers.
767///
768/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm256_cvtepi32_epi64)
769#[inline]
770#[target_feature(enable = "avx2")]
771#[cfg_attr(test, assert_instr(vpmovsxdq))]
772#[stable(feature = "simd_x86", since = "1.27.0")]
773#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")]
774pub const fn _mm256_cvtepi32_epi64(a: __m128i) -> __m256i {
775    unsafe { transmute::<i64x4, _>(simd_cast(a.as_i32x4())) }
776}
777
778/// Sign-extend 8-bit integers to 16-bit integers.
779///
780/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm256_cvtepi8_epi16)
781#[inline]
782#[target_feature(enable = "avx2")]
783#[cfg_attr(test, assert_instr(vpmovsxbw))]
784#[stable(feature = "simd_x86", since = "1.27.0")]
785#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")]
786pub const fn _mm256_cvtepi8_epi16(a: __m128i) -> __m256i {
787    unsafe { transmute::<i16x16, _>(simd_cast(a.as_i8x16())) }
788}
789
790/// Sign-extend 8-bit integers to 32-bit integers.
791///
792/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm256_cvtepi8_epi32)
793#[inline]
794#[target_feature(enable = "avx2")]
795#[cfg_attr(test, assert_instr(vpmovsxbd))]
796#[stable(feature = "simd_x86", since = "1.27.0")]
797#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")]
798pub const fn _mm256_cvtepi8_epi32(a: __m128i) -> __m256i {
799    unsafe {
800        let a = a.as_i8x16();
801        let v64: i8x8 = simd_shuffle!(a, a, [0, 1, 2, 3, 4, 5, 6, 7]);
802        transmute::<i32x8, _>(simd_cast(v64))
803    }
804}
805
806/// Sign-extend 8-bit integers to 64-bit integers.
807///
808/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm256_cvtepi8_epi64)
809#[inline]
810#[target_feature(enable = "avx2")]
811#[cfg_attr(test, assert_instr(vpmovsxbq))]
812#[stable(feature = "simd_x86", since = "1.27.0")]
813#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")]
814pub const fn _mm256_cvtepi8_epi64(a: __m128i) -> __m256i {
815    unsafe {
816        let a = a.as_i8x16();
817        let v32: i8x4 = simd_shuffle!(a, a, [0, 1, 2, 3]);
818        transmute::<i64x4, _>(simd_cast(v32))
819    }
820}
821
822/// Zeroes extend packed unsigned 16-bit integers in `a` to packed 32-bit
823/// integers, and stores the results in `dst`.
824///
825/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm256_cvtepu16_epi32)
826#[inline]
827#[target_feature(enable = "avx2")]
828#[cfg_attr(test, assert_instr(vpmovzxwd))]
829#[stable(feature = "simd_x86", since = "1.27.0")]
830#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")]
831pub const fn _mm256_cvtepu16_epi32(a: __m128i) -> __m256i {
832    unsafe { transmute::<i32x8, _>(simd_cast(a.as_u16x8())) }
833}
834
835/// Zero-extend the lower four unsigned 16-bit integers in `a` to 64-bit
836/// integers. The upper four elements of `a` are unused.
837///
838/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm256_cvtepu16_epi64)
839#[inline]
840#[target_feature(enable = "avx2")]
841#[cfg_attr(test, assert_instr(vpmovzxwq))]
842#[stable(feature = "simd_x86", since = "1.27.0")]
843#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")]
844pub const fn _mm256_cvtepu16_epi64(a: __m128i) -> __m256i {
845    unsafe {
846        let a = a.as_u16x8();
847        let v64: u16x4 = simd_shuffle!(a, a, [0, 1, 2, 3]);
848        transmute::<i64x4, _>(simd_cast(v64))
849    }
850}
851
852/// Zero-extend unsigned 32-bit integers in `a` to 64-bit integers.
853///
854/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm256_cvtepu32_epi64)
855#[inline]
856#[target_feature(enable = "avx2")]
857#[cfg_attr(test, assert_instr(vpmovzxdq))]
858#[stable(feature = "simd_x86", since = "1.27.0")]
859#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")]
860pub const fn _mm256_cvtepu32_epi64(a: __m128i) -> __m256i {
861    unsafe { transmute::<i64x4, _>(simd_cast(a.as_u32x4())) }
862}
863
864/// Zero-extend unsigned 8-bit integers in `a` to 16-bit integers.
865///
866/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm256_cvtepu8_epi16)
867#[inline]
868#[target_feature(enable = "avx2")]
869#[cfg_attr(test, assert_instr(vpmovzxbw))]
870#[stable(feature = "simd_x86", since = "1.27.0")]
871#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")]
872pub const fn _mm256_cvtepu8_epi16(a: __m128i) -> __m256i {
873    unsafe { transmute::<i16x16, _>(simd_cast(a.as_u8x16())) }
874}
875
876/// Zero-extend the lower eight unsigned 8-bit integers in `a` to 32-bit
877/// integers. The upper eight elements of `a` are unused.
878///
879/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm256_cvtepu8_epi32)
880#[inline]
881#[target_feature(enable = "avx2")]
882#[cfg_attr(test, assert_instr(vpmovzxbd))]
883#[stable(feature = "simd_x86", since = "1.27.0")]
884#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")]
885pub const fn _mm256_cvtepu8_epi32(a: __m128i) -> __m256i {
886    unsafe {
887        let a = a.as_u8x16();
888        let v64: u8x8 = simd_shuffle!(a, a, [0, 1, 2, 3, 4, 5, 6, 7]);
889        transmute::<i32x8, _>(simd_cast(v64))
890    }
891}
892
893/// Zero-extend the lower four unsigned 8-bit integers in `a` to 64-bit
894/// integers. The upper twelve elements of `a` are unused.
895///
896/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm256_cvtepu8_epi64)
897#[inline]
898#[target_feature(enable = "avx2")]
899#[cfg_attr(test, assert_instr(vpmovzxbq))]
900#[stable(feature = "simd_x86", since = "1.27.0")]
901#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")]
902pub const fn _mm256_cvtepu8_epi64(a: __m128i) -> __m256i {
903    unsafe {
904        let a = a.as_u8x16();
905        let v32: u8x4 = simd_shuffle!(a, a, [0, 1, 2, 3]);
906        transmute::<i64x4, _>(simd_cast(v32))
907    }
908}
909
910/// Extracts 128 bits (of integer data) from `a` selected with `IMM1`.
911///
912/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm256_extracti128_si256)
913#[inline]
914#[target_feature(enable = "avx2")]
915#[cfg_attr(test, assert_instr(vextractf128, IMM1 = 1))]
916#[rustc_legacy_const_generics(1)]
917#[stable(feature = "simd_x86", since = "1.27.0")]
918#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")]
919pub const fn _mm256_extracti128_si256<const IMM1: i32>(a: __m256i) -> __m128i {
920    static_assert_uimm_bits!(IMM1, 1);
921    unsafe {
922        let a = a.as_i64x4();
923        let b = i64x4::ZERO;
924        let dst: i64x2 = simd_shuffle!(a, b, [[0, 1], [2, 3]][IMM1 as usize]);
925        transmute(dst)
926    }
927}
928
929/// Horizontally adds adjacent pairs of 16-bit integers in `a` and `b`.
930///
931/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm256_hadd_epi16)
932#[inline]
933#[target_feature(enable = "avx2")]
934#[cfg_attr(test, assert_instr(vphaddw))]
935#[stable(feature = "simd_x86", since = "1.27.0")]
936#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")]
937pub const fn _mm256_hadd_epi16(a: __m256i, b: __m256i) -> __m256i {
938    let a = a.as_i16x16();
939    let b = b.as_i16x16();
940    unsafe {
941        let even: i16x16 = simd_shuffle!(
942            a,
943            b,
944            [0, 2, 4, 6, 16, 18, 20, 22, 8, 10, 12, 14, 24, 26, 28, 30]
945        );
946        let odd: i16x16 = simd_shuffle!(
947            a,
948            b,
949            [1, 3, 5, 7, 17, 19, 21, 23, 9, 11, 13, 15, 25, 27, 29, 31]
950        );
951        simd_add(even, odd).as_m256i()
952    }
953}
954
955/// Horizontally adds adjacent pairs of 32-bit integers in `a` and `b`.
956///
957/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm256_hadd_epi32)
958#[inline]
959#[target_feature(enable = "avx2")]
960#[cfg_attr(test, assert_instr(vphaddd))]
961#[stable(feature = "simd_x86", since = "1.27.0")]
962#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")]
963pub const fn _mm256_hadd_epi32(a: __m256i, b: __m256i) -> __m256i {
964    let a = a.as_i32x8();
965    let b = b.as_i32x8();
966    unsafe {
967        let even: i32x8 = simd_shuffle!(a, b, [0, 2, 8, 10, 4, 6, 12, 14]);
968        let odd: i32x8 = simd_shuffle!(a, b, [1, 3, 9, 11, 5, 7, 13, 15]);
969        simd_add(even, odd).as_m256i()
970    }
971}
972
973/// Horizontally adds adjacent pairs of 16-bit integers in `a` and `b`
974/// using saturation.
975///
976/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm256_hadds_epi16)
977#[inline]
978#[target_feature(enable = "avx2")]
979#[cfg_attr(test, assert_instr(vphaddsw))]
980#[stable(feature = "simd_x86", since = "1.27.0")]
981pub fn _mm256_hadds_epi16(a: __m256i, b: __m256i) -> __m256i {
982    let a = a.as_i16x16();
983    let b = b.as_i16x16();
984    unsafe {
985        let even: i16x16 = simd_shuffle!(
986            a,
987            b,
988            [0, 2, 4, 6, 16, 18, 20, 22, 8, 10, 12, 14, 24, 26, 28, 30]
989        );
990        let odd: i16x16 = simd_shuffle!(
991            a,
992            b,
993            [1, 3, 5, 7, 17, 19, 21, 23, 9, 11, 13, 15, 25, 27, 29, 31]
994        );
995        simd_saturating_add(even, odd).as_m256i()
996    }
997}
998
999/// Horizontally subtract adjacent pairs of 16-bit integers in `a` and `b`.
1000///
1001/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm256_hsub_epi16)
1002#[inline]
1003#[target_feature(enable = "avx2")]
1004#[cfg_attr(test, assert_instr(vphsubw))]
1005#[stable(feature = "simd_x86", since = "1.27.0")]
1006#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")]
1007pub const fn _mm256_hsub_epi16(a: __m256i, b: __m256i) -> __m256i {
1008    let a = a.as_i16x16();
1009    let b = b.as_i16x16();
1010    unsafe {
1011        let even: i16x16 = simd_shuffle!(
1012            a,
1013            b,
1014            [0, 2, 4, 6, 16, 18, 20, 22, 8, 10, 12, 14, 24, 26, 28, 30]
1015        );
1016        let odd: i16x16 = simd_shuffle!(
1017            a,
1018            b,
1019            [1, 3, 5, 7, 17, 19, 21, 23, 9, 11, 13, 15, 25, 27, 29, 31]
1020        );
1021        simd_sub(even, odd).as_m256i()
1022    }
1023}
1024
1025/// Horizontally subtract adjacent pairs of 32-bit integers in `a` and `b`.
1026///
1027/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm256_hsub_epi32)
1028#[inline]
1029#[target_feature(enable = "avx2")]
1030#[cfg_attr(test, assert_instr(vphsubd))]
1031#[stable(feature = "simd_x86", since = "1.27.0")]
1032#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")]
1033pub const fn _mm256_hsub_epi32(a: __m256i, b: __m256i) -> __m256i {
1034    let a = a.as_i32x8();
1035    let b = b.as_i32x8();
1036    unsafe {
1037        let even: i32x8 = simd_shuffle!(a, b, [0, 2, 8, 10, 4, 6, 12, 14]);
1038        let odd: i32x8 = simd_shuffle!(a, b, [1, 3, 9, 11, 5, 7, 13, 15]);
1039        simd_sub(even, odd).as_m256i()
1040    }
1041}
1042
1043/// Horizontally subtract adjacent pairs of 16-bit integers in `a` and `b`
1044/// using saturation.
1045///
1046/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm256_hsubs_epi16)
1047#[inline]
1048#[target_feature(enable = "avx2")]
1049#[cfg_attr(test, assert_instr(vphsubsw))]
1050#[stable(feature = "simd_x86", since = "1.27.0")]
1051pub fn _mm256_hsubs_epi16(a: __m256i, b: __m256i) -> __m256i {
1052    let a = a.as_i16x16();
1053    let b = b.as_i16x16();
1054    unsafe {
1055        let even: i16x16 = simd_shuffle!(
1056            a,
1057            b,
1058            [0, 2, 4, 6, 16, 18, 20, 22, 8, 10, 12, 14, 24, 26, 28, 30]
1059        );
1060        let odd: i16x16 = simd_shuffle!(
1061            a,
1062            b,
1063            [1, 3, 5, 7, 17, 19, 21, 23, 9, 11, 13, 15, 25, 27, 29, 31]
1064        );
1065        simd_saturating_sub(even, odd).as_m256i()
1066    }
1067}
1068
1069/// Returns values from `slice` at offsets determined by `offsets * scale`,
1070/// where
1071/// `scale` should be 1, 2, 4 or 8.
1072///
1073/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_i32gather_epi32)
1074#[inline]
1075#[target_feature(enable = "avx2")]
1076#[cfg_attr(test, assert_instr(vpgatherdd, SCALE = 1))]
1077#[rustc_legacy_const_generics(2)]
1078#[stable(feature = "simd_x86", since = "1.27.0")]
1079pub unsafe fn _mm_i32gather_epi32<const SCALE: i32>(
1080    slice: *const i32,
1081    offsets: __m128i,
1082) -> __m128i {
1083    static_assert_imm8_scale!(SCALE);
1084    let zero = i32x4::ZERO;
1085    let neg_one = _mm_set1_epi32(-1).as_i32x4();
1086    let offsets = offsets.as_i32x4();
1087    let slice = slice as *const i8;
1088    let r = pgatherdd(zero, slice, offsets, neg_one, SCALE as i8);
1089    transmute(r)
1090}
1091
1092/// Returns values from `slice` at offsets determined by `offsets * scale`,
1093/// where
1094/// `scale` should be 1, 2, 4 or 8. If mask is set, load the value from `src` in
1095/// that position instead.
1096///
1097/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_mask_i32gather_epi32)
1098#[inline]
1099#[target_feature(enable = "avx2")]
1100#[cfg_attr(test, assert_instr(vpgatherdd, SCALE = 1))]
1101#[rustc_legacy_const_generics(4)]
1102#[stable(feature = "simd_x86", since = "1.27.0")]
1103pub unsafe fn _mm_mask_i32gather_epi32<const SCALE: i32>(
1104    src: __m128i,
1105    slice: *const i32,
1106    offsets: __m128i,
1107    mask: __m128i,
1108) -> __m128i {
1109    static_assert_imm8_scale!(SCALE);
1110    let src = src.as_i32x4();
1111    let mask = mask.as_i32x4();
1112    let offsets = offsets.as_i32x4();
1113    let slice = slice as *const i8;
1114    let r = pgatherdd(src, slice, offsets, mask, SCALE as i8);
1115    transmute(r)
1116}
1117
1118/// Returns values from `slice` at offsets determined by `offsets * scale`,
1119/// where
1120/// `scale` should be 1, 2, 4 or 8.
1121///
1122/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm256_i32gather_epi32)
1123#[inline]
1124#[target_feature(enable = "avx2")]
1125#[cfg_attr(test, assert_instr(vpgatherdd, SCALE = 1))]
1126#[rustc_legacy_const_generics(2)]
1127#[stable(feature = "simd_x86", since = "1.27.0")]
1128pub unsafe fn _mm256_i32gather_epi32<const SCALE: i32>(
1129    slice: *const i32,
1130    offsets: __m256i,
1131) -> __m256i {
1132    static_assert_imm8_scale!(SCALE);
1133    let zero = i32x8::ZERO;
1134    let neg_one = _mm256_set1_epi32(-1).as_i32x8();
1135    let offsets = offsets.as_i32x8();
1136    let slice = slice as *const i8;
1137    let r = vpgatherdd(zero, slice, offsets, neg_one, SCALE as i8);
1138    transmute(r)
1139}
1140
1141/// Returns values from `slice` at offsets determined by `offsets * scale`,
1142/// where
1143/// `scale` should be 1, 2, 4 or 8. If mask is set, load the value from `src` in
1144/// that position instead.
1145///
1146/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm256_mask_i32gather_epi32)
1147#[inline]
1148#[target_feature(enable = "avx2")]
1149#[cfg_attr(test, assert_instr(vpgatherdd, SCALE = 1))]
1150#[rustc_legacy_const_generics(4)]
1151#[stable(feature = "simd_x86", since = "1.27.0")]
1152pub unsafe fn _mm256_mask_i32gather_epi32<const SCALE: i32>(
1153    src: __m256i,
1154    slice: *const i32,
1155    offsets: __m256i,
1156    mask: __m256i,
1157) -> __m256i {
1158    static_assert_imm8_scale!(SCALE);
1159    let src = src.as_i32x8();
1160    let mask = mask.as_i32x8();
1161    let offsets = offsets.as_i32x8();
1162    let slice = slice as *const i8;
1163    let r = vpgatherdd(src, slice, offsets, mask, SCALE as i8);
1164    transmute(r)
1165}
1166
1167/// Returns values from `slice` at offsets determined by `offsets * scale`,
1168/// where
1169/// `scale` should be 1, 2, 4 or 8.
1170///
1171/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_i32gather_ps)
1172#[inline]
1173#[target_feature(enable = "avx2")]
1174#[cfg_attr(test, assert_instr(vgatherdps, SCALE = 1))]
1175#[rustc_legacy_const_generics(2)]
1176#[stable(feature = "simd_x86", since = "1.27.0")]
1177pub unsafe fn _mm_i32gather_ps<const SCALE: i32>(slice: *const f32, offsets: __m128i) -> __m128 {
1178    static_assert_imm8_scale!(SCALE);
1179    let zero = _mm_setzero_ps();
1180    let neg_one = _mm_set1_ps(-1.0);
1181    let offsets = offsets.as_i32x4();
1182    let slice = slice as *const i8;
1183    pgatherdps(zero, slice, offsets, neg_one, SCALE as i8)
1184}
1185
1186/// Returns values from `slice` at offsets determined by `offsets * scale`,
1187/// where
1188/// `scale` should be 1, 2, 4 or 8. If mask is set, load the value from `src` in
1189/// that position instead.
1190///
1191/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_mask_i32gather_ps)
1192#[inline]
1193#[target_feature(enable = "avx2")]
1194#[cfg_attr(test, assert_instr(vgatherdps, SCALE = 1))]
1195#[rustc_legacy_const_generics(4)]
1196#[stable(feature = "simd_x86", since = "1.27.0")]
1197pub unsafe fn _mm_mask_i32gather_ps<const SCALE: i32>(
1198    src: __m128,
1199    slice: *const f32,
1200    offsets: __m128i,
1201    mask: __m128,
1202) -> __m128 {
1203    static_assert_imm8_scale!(SCALE);
1204    let offsets = offsets.as_i32x4();
1205    let slice = slice as *const i8;
1206    pgatherdps(src, slice, offsets, mask, SCALE as i8)
1207}
1208
1209/// Returns values from `slice` at offsets determined by `offsets * scale`,
1210/// where
1211/// `scale` should be 1, 2, 4 or 8.
1212///
1213/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm256_i32gather_ps)
1214#[inline]
1215#[target_feature(enable = "avx2")]
1216#[cfg_attr(test, assert_instr(vgatherdps, SCALE = 1))]
1217#[rustc_legacy_const_generics(2)]
1218#[stable(feature = "simd_x86", since = "1.27.0")]
1219pub unsafe fn _mm256_i32gather_ps<const SCALE: i32>(slice: *const f32, offsets: __m256i) -> __m256 {
1220    static_assert_imm8_scale!(SCALE);
1221    let zero = _mm256_setzero_ps();
1222    let neg_one = _mm256_set1_ps(-1.0);
1223    let offsets = offsets.as_i32x8();
1224    let slice = slice as *const i8;
1225    vpgatherdps(zero, slice, offsets, neg_one, SCALE as i8)
1226}
1227
1228/// Returns values from `slice` at offsets determined by `offsets * scale`,
1229/// where
1230/// `scale` should be 1, 2, 4 or 8. If mask is set, load the value from `src` in
1231/// that position instead.
1232///
1233/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm256_mask_i32gather_ps)
1234#[inline]
1235#[target_feature(enable = "avx2")]
1236#[cfg_attr(test, assert_instr(vgatherdps, SCALE = 1))]
1237#[rustc_legacy_const_generics(4)]
1238#[stable(feature = "simd_x86", since = "1.27.0")]
1239pub unsafe fn _mm256_mask_i32gather_ps<const SCALE: i32>(
1240    src: __m256,
1241    slice: *const f32,
1242    offsets: __m256i,
1243    mask: __m256,
1244) -> __m256 {
1245    static_assert_imm8_scale!(SCALE);
1246    let offsets = offsets.as_i32x8();
1247    let slice = slice as *const i8;
1248    vpgatherdps(src, slice, offsets, mask, SCALE as i8)
1249}
1250
1251/// Returns values from `slice` at offsets determined by `offsets * scale`,
1252/// where
1253/// `scale` should be 1, 2, 4 or 8.
1254///
1255/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_i32gather_epi64)
1256#[inline]
1257#[target_feature(enable = "avx2")]
1258#[cfg_attr(test, assert_instr(vpgatherdq, SCALE = 1))]
1259#[rustc_legacy_const_generics(2)]
1260#[stable(feature = "simd_x86", since = "1.27.0")]
1261pub unsafe fn _mm_i32gather_epi64<const SCALE: i32>(
1262    slice: *const i64,
1263    offsets: __m128i,
1264) -> __m128i {
1265    static_assert_imm8_scale!(SCALE);
1266    let zero = i64x2::ZERO;
1267    let neg_one = _mm_set1_epi64x(-1).as_i64x2();
1268    let offsets = offsets.as_i32x4();
1269    let slice = slice as *const i8;
1270    let r = pgatherdq(zero, slice, offsets, neg_one, SCALE as i8);
1271    transmute(r)
1272}
1273
1274/// Returns values from `slice` at offsets determined by `offsets * scale`,
1275/// where
1276/// `scale` should be 1, 2, 4 or 8. If mask is set, load the value from `src` in
1277/// that position instead.
1278///
1279/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_mask_i32gather_epi64)
1280#[inline]
1281#[target_feature(enable = "avx2")]
1282#[cfg_attr(test, assert_instr(vpgatherdq, SCALE = 1))]
1283#[rustc_legacy_const_generics(4)]
1284#[stable(feature = "simd_x86", since = "1.27.0")]
1285pub unsafe fn _mm_mask_i32gather_epi64<const SCALE: i32>(
1286    src: __m128i,
1287    slice: *const i64,
1288    offsets: __m128i,
1289    mask: __m128i,
1290) -> __m128i {
1291    static_assert_imm8_scale!(SCALE);
1292    let src = src.as_i64x2();
1293    let mask = mask.as_i64x2();
1294    let offsets = offsets.as_i32x4();
1295    let slice = slice as *const i8;
1296    let r = pgatherdq(src, slice, offsets, mask, SCALE as i8);
1297    transmute(r)
1298}
1299
1300/// Returns values from `slice` at offsets determined by `offsets * scale`,
1301/// where
1302/// `scale` should be 1, 2, 4 or 8.
1303///
1304/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm256_i32gather_epi64)
1305#[inline]
1306#[target_feature(enable = "avx2")]
1307#[cfg_attr(test, assert_instr(vpgatherdq, SCALE = 1))]
1308#[rustc_legacy_const_generics(2)]
1309#[stable(feature = "simd_x86", since = "1.27.0")]
1310pub unsafe fn _mm256_i32gather_epi64<const SCALE: i32>(
1311    slice: *const i64,
1312    offsets: __m128i,
1313) -> __m256i {
1314    static_assert_imm8_scale!(SCALE);
1315    let zero = i64x4::ZERO;
1316    let neg_one = _mm256_set1_epi64x(-1).as_i64x4();
1317    let offsets = offsets.as_i32x4();
1318    let slice = slice as *const i8;
1319    let r = vpgatherdq(zero, slice, offsets, neg_one, SCALE as i8);
1320    transmute(r)
1321}
1322
1323/// Returns values from `slice` at offsets determined by `offsets * scale`,
1324/// where
1325/// `scale` should be 1, 2, 4 or 8. If mask is set, load the value from `src` in
1326/// that position instead.
1327///
1328/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm256_mask_i32gather_epi64)
1329#[inline]
1330#[target_feature(enable = "avx2")]
1331#[cfg_attr(test, assert_instr(vpgatherdq, SCALE = 1))]
1332#[rustc_legacy_const_generics(4)]
1333#[stable(feature = "simd_x86", since = "1.27.0")]
1334pub unsafe fn _mm256_mask_i32gather_epi64<const SCALE: i32>(
1335    src: __m256i,
1336    slice: *const i64,
1337    offsets: __m128i,
1338    mask: __m256i,
1339) -> __m256i {
1340    static_assert_imm8_scale!(SCALE);
1341    let src = src.as_i64x4();
1342    let mask = mask.as_i64x4();
1343    let offsets = offsets.as_i32x4();
1344    let slice = slice as *const i8;
1345    let r = vpgatherdq(src, slice, offsets, mask, SCALE as i8);
1346    transmute(r)
1347}
1348
1349/// Returns values from `slice` at offsets determined by `offsets * scale`,
1350/// where
1351/// `scale` should be 1, 2, 4 or 8.
1352///
1353/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_i32gather_pd)
1354#[inline]
1355#[target_feature(enable = "avx2")]
1356#[cfg_attr(test, assert_instr(vgatherdpd, SCALE = 1))]
1357#[rustc_legacy_const_generics(2)]
1358#[stable(feature = "simd_x86", since = "1.27.0")]
1359pub unsafe fn _mm_i32gather_pd<const SCALE: i32>(slice: *const f64, offsets: __m128i) -> __m128d {
1360    static_assert_imm8_scale!(SCALE);
1361    let zero = _mm_setzero_pd();
1362    let neg_one = _mm_set1_pd(-1.0);
1363    let offsets = offsets.as_i32x4();
1364    let slice = slice as *const i8;
1365    pgatherdpd(zero, slice, offsets, neg_one, SCALE as i8)
1366}
1367
1368/// Returns values from `slice` at offsets determined by `offsets * scale`,
1369/// where
1370/// `scale` should be 1, 2, 4 or 8. If mask is set, load the value from `src` in
1371/// that position instead.
1372///
1373/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_mask_i32gather_pd)
1374#[inline]
1375#[target_feature(enable = "avx2")]
1376#[cfg_attr(test, assert_instr(vgatherdpd, SCALE = 1))]
1377#[rustc_legacy_const_generics(4)]
1378#[stable(feature = "simd_x86", since = "1.27.0")]
1379pub unsafe fn _mm_mask_i32gather_pd<const SCALE: i32>(
1380    src: __m128d,
1381    slice: *const f64,
1382    offsets: __m128i,
1383    mask: __m128d,
1384) -> __m128d {
1385    static_assert_imm8_scale!(SCALE);
1386    let offsets = offsets.as_i32x4();
1387    let slice = slice as *const i8;
1388    pgatherdpd(src, slice, offsets, mask, SCALE as i8)
1389}
1390
1391/// Returns values from `slice` at offsets determined by `offsets * scale`,
1392/// where
1393/// `scale` should be 1, 2, 4 or 8.
1394///
1395/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm256_i32gather_pd)
1396#[inline]
1397#[target_feature(enable = "avx2")]
1398#[cfg_attr(test, assert_instr(vgatherdpd, SCALE = 1))]
1399#[rustc_legacy_const_generics(2)]
1400#[stable(feature = "simd_x86", since = "1.27.0")]
1401pub unsafe fn _mm256_i32gather_pd<const SCALE: i32>(
1402    slice: *const f64,
1403    offsets: __m128i,
1404) -> __m256d {
1405    static_assert_imm8_scale!(SCALE);
1406    let zero = _mm256_setzero_pd();
1407    let neg_one = _mm256_set1_pd(-1.0);
1408    let offsets = offsets.as_i32x4();
1409    let slice = slice as *const i8;
1410    vpgatherdpd(zero, slice, offsets, neg_one, SCALE as i8)
1411}
1412
1413/// Returns values from `slice` at offsets determined by `offsets * scale`,
1414/// where
1415/// `scale` should be 1, 2, 4 or 8. If mask is set, load the value from `src` in
1416/// that position instead.
1417///
1418/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm256_mask_i32gather_pd)
1419#[inline]
1420#[target_feature(enable = "avx2")]
1421#[cfg_attr(test, assert_instr(vgatherdpd, SCALE = 1))]
1422#[rustc_legacy_const_generics(4)]
1423#[stable(feature = "simd_x86", since = "1.27.0")]
1424pub unsafe fn _mm256_mask_i32gather_pd<const SCALE: i32>(
1425    src: __m256d,
1426    slice: *const f64,
1427    offsets: __m128i,
1428    mask: __m256d,
1429) -> __m256d {
1430    static_assert_imm8_scale!(SCALE);
1431    let offsets = offsets.as_i32x4();
1432    let slice = slice as *const i8;
1433    vpgatherdpd(src, slice, offsets, mask, SCALE as i8)
1434}
1435
1436/// Returns values from `slice` at offsets determined by `offsets * scale`,
1437/// where
1438/// `scale` should be 1, 2, 4 or 8.
1439///
1440/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_i64gather_epi32)
1441#[inline]
1442#[target_feature(enable = "avx2")]
1443#[cfg_attr(test, assert_instr(vpgatherqd, SCALE = 1))]
1444#[rustc_legacy_const_generics(2)]
1445#[stable(feature = "simd_x86", since = "1.27.0")]
1446pub unsafe fn _mm_i64gather_epi32<const SCALE: i32>(
1447    slice: *const i32,
1448    offsets: __m128i,
1449) -> __m128i {
1450    static_assert_imm8_scale!(SCALE);
1451    let zero = i32x4::ZERO;
1452    let neg_one = _mm_set1_epi64x(-1).as_i32x4();
1453    let offsets = offsets.as_i64x2();
1454    let slice = slice as *const i8;
1455    let r = pgatherqd(zero, slice, offsets, neg_one, SCALE as i8);
1456    transmute(r)
1457}
1458
1459/// Returns values from `slice` at offsets determined by `offsets * scale`,
1460/// where
1461/// `scale` should be 1, 2, 4 or 8. If mask is set, load the value from `src` in
1462/// that position instead.
1463///
1464/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_mask_i64gather_epi32)
1465#[inline]
1466#[target_feature(enable = "avx2")]
1467#[cfg_attr(test, assert_instr(vpgatherqd, SCALE = 1))]
1468#[rustc_legacy_const_generics(4)]
1469#[stable(feature = "simd_x86", since = "1.27.0")]
1470pub unsafe fn _mm_mask_i64gather_epi32<const SCALE: i32>(
1471    src: __m128i,
1472    slice: *const i32,
1473    offsets: __m128i,
1474    mask: __m128i,
1475) -> __m128i {
1476    static_assert_imm8_scale!(SCALE);
1477    let src = src.as_i32x4();
1478    let mask = mask.as_i32x4();
1479    let offsets = offsets.as_i64x2();
1480    let slice = slice as *const i8;
1481    let r = pgatherqd(src, slice, offsets, mask, SCALE as i8);
1482    transmute(r)
1483}
1484
1485/// Returns values from `slice` at offsets determined by `offsets * scale`,
1486/// where
1487/// `scale` should be 1, 2, 4 or 8.
1488///
1489/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm256_i64gather_epi32)
1490#[inline]
1491#[target_feature(enable = "avx2")]
1492#[cfg_attr(test, assert_instr(vpgatherqd, SCALE = 1))]
1493#[rustc_legacy_const_generics(2)]
1494#[stable(feature = "simd_x86", since = "1.27.0")]
1495pub unsafe fn _mm256_i64gather_epi32<const SCALE: i32>(
1496    slice: *const i32,
1497    offsets: __m256i,
1498) -> __m128i {
1499    static_assert_imm8_scale!(SCALE);
1500    let zero = i32x4::ZERO;
1501    let neg_one = _mm_set1_epi64x(-1).as_i32x4();
1502    let offsets = offsets.as_i64x4();
1503    let slice = slice as *const i8;
1504    let r = vpgatherqd(zero, slice, offsets, neg_one, SCALE as i8);
1505    transmute(r)
1506}
1507
1508/// Returns values from `slice` at offsets determined by `offsets * scale`,
1509/// where
1510/// `scale` should be 1, 2, 4 or 8. If mask is set, load the value from `src` in
1511/// that position instead.
1512///
1513/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm256_mask_i64gather_epi32)
1514#[inline]
1515#[target_feature(enable = "avx2")]
1516#[cfg_attr(test, assert_instr(vpgatherqd, SCALE = 1))]
1517#[rustc_legacy_const_generics(4)]
1518#[stable(feature = "simd_x86", since = "1.27.0")]
1519pub unsafe fn _mm256_mask_i64gather_epi32<const SCALE: i32>(
1520    src: __m128i,
1521    slice: *const i32,
1522    offsets: __m256i,
1523    mask: __m128i,
1524) -> __m128i {
1525    static_assert_imm8_scale!(SCALE);
1526    let src = src.as_i32x4();
1527    let mask = mask.as_i32x4();
1528    let offsets = offsets.as_i64x4();
1529    let slice = slice as *const i8;
1530    let r = vpgatherqd(src, slice, offsets, mask, SCALE as i8);
1531    transmute(r)
1532}
1533
1534/// Returns values from `slice` at offsets determined by `offsets * scale`,
1535/// where
1536/// `scale` should be 1, 2, 4 or 8.
1537///
1538/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_i64gather_ps)
1539#[inline]
1540#[target_feature(enable = "avx2")]
1541#[cfg_attr(test, assert_instr(vgatherqps, SCALE = 1))]
1542#[rustc_legacy_const_generics(2)]
1543#[stable(feature = "simd_x86", since = "1.27.0")]
1544pub unsafe fn _mm_i64gather_ps<const SCALE: i32>(slice: *const f32, offsets: __m128i) -> __m128 {
1545    static_assert_imm8_scale!(SCALE);
1546    let zero = _mm_setzero_ps();
1547    let neg_one = _mm_set1_ps(-1.0);
1548    let offsets = offsets.as_i64x2();
1549    let slice = slice as *const i8;
1550    pgatherqps(zero, slice, offsets, neg_one, SCALE as i8)
1551}
1552
1553/// Returns values from `slice` at offsets determined by `offsets * scale`,
1554/// where
1555/// `scale` should be 1, 2, 4 or 8. If mask is set, load the value from `src` in
1556/// that position instead.
1557///
1558/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_mask_i64gather_ps)
1559#[inline]
1560#[target_feature(enable = "avx2")]
1561#[cfg_attr(test, assert_instr(vgatherqps, SCALE = 1))]
1562#[rustc_legacy_const_generics(4)]
1563#[stable(feature = "simd_x86", since = "1.27.0")]
1564pub unsafe fn _mm_mask_i64gather_ps<const SCALE: i32>(
1565    src: __m128,
1566    slice: *const f32,
1567    offsets: __m128i,
1568    mask: __m128,
1569) -> __m128 {
1570    static_assert_imm8_scale!(SCALE);
1571    let offsets = offsets.as_i64x2();
1572    let slice = slice as *const i8;
1573    pgatherqps(src, slice, offsets, mask, SCALE as i8)
1574}
1575
1576/// Returns values from `slice` at offsets determined by `offsets * scale`,
1577/// where
1578/// `scale` should be 1, 2, 4 or 8.
1579///
1580/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm256_i64gather_ps)
1581#[inline]
1582#[target_feature(enable = "avx2")]
1583#[cfg_attr(test, assert_instr(vgatherqps, SCALE = 1))]
1584#[rustc_legacy_const_generics(2)]
1585#[stable(feature = "simd_x86", since = "1.27.0")]
1586pub unsafe fn _mm256_i64gather_ps<const SCALE: i32>(slice: *const f32, offsets: __m256i) -> __m128 {
1587    static_assert_imm8_scale!(SCALE);
1588    let zero = _mm_setzero_ps();
1589    let neg_one = _mm_set1_ps(-1.0);
1590    let offsets = offsets.as_i64x4();
1591    let slice = slice as *const i8;
1592    vpgatherqps(zero, slice, offsets, neg_one, SCALE as i8)
1593}
1594
1595/// Returns values from `slice` at offsets determined by `offsets * scale`,
1596/// where
1597/// `scale` should be 1, 2, 4 or 8. If mask is set, load the value from `src` in
1598/// that position instead.
1599///
1600/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm256_mask_i64gather_ps)
1601#[inline]
1602#[target_feature(enable = "avx2")]
1603#[cfg_attr(test, assert_instr(vgatherqps, SCALE = 1))]
1604#[rustc_legacy_const_generics(4)]
1605#[stable(feature = "simd_x86", since = "1.27.0")]
1606pub unsafe fn _mm256_mask_i64gather_ps<const SCALE: i32>(
1607    src: __m128,
1608    slice: *const f32,
1609    offsets: __m256i,
1610    mask: __m128,
1611) -> __m128 {
1612    static_assert_imm8_scale!(SCALE);
1613    let offsets = offsets.as_i64x4();
1614    let slice = slice as *const i8;
1615    vpgatherqps(src, slice, offsets, mask, SCALE as i8)
1616}
1617
1618/// Returns values from `slice` at offsets determined by `offsets * scale`,
1619/// where
1620/// `scale` should be 1, 2, 4 or 8.
1621///
1622/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_i64gather_epi64)
1623#[inline]
1624#[target_feature(enable = "avx2")]
1625#[cfg_attr(test, assert_instr(vpgatherqq, SCALE = 1))]
1626#[rustc_legacy_const_generics(2)]
1627#[stable(feature = "simd_x86", since = "1.27.0")]
1628pub unsafe fn _mm_i64gather_epi64<const SCALE: i32>(
1629    slice: *const i64,
1630    offsets: __m128i,
1631) -> __m128i {
1632    static_assert_imm8_scale!(SCALE);
1633    let zero = i64x2::ZERO;
1634    let neg_one = _mm_set1_epi64x(-1).as_i64x2();
1635    let slice = slice as *const i8;
1636    let offsets = offsets.as_i64x2();
1637    let r = pgatherqq(zero, slice, offsets, neg_one, SCALE as i8);
1638    transmute(r)
1639}
1640
1641/// Returns values from `slice` at offsets determined by `offsets * scale`,
1642/// where
1643/// `scale` should be 1, 2, 4 or 8. If mask is set, load the value from `src` in
1644/// that position instead.
1645///
1646/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_mask_i64gather_epi64)
1647#[inline]
1648#[target_feature(enable = "avx2")]
1649#[cfg_attr(test, assert_instr(vpgatherqq, SCALE = 1))]
1650#[rustc_legacy_const_generics(4)]
1651#[stable(feature = "simd_x86", since = "1.27.0")]
1652pub unsafe fn _mm_mask_i64gather_epi64<const SCALE: i32>(
1653    src: __m128i,
1654    slice: *const i64,
1655    offsets: __m128i,
1656    mask: __m128i,
1657) -> __m128i {
1658    static_assert_imm8_scale!(SCALE);
1659    let src = src.as_i64x2();
1660    let mask = mask.as_i64x2();
1661    let offsets = offsets.as_i64x2();
1662    let slice = slice as *const i8;
1663    let r = pgatherqq(src, slice, offsets, mask, SCALE as i8);
1664    transmute(r)
1665}
1666
1667/// Returns values from `slice` at offsets determined by `offsets * scale`,
1668/// where
1669/// `scale` should be 1, 2, 4 or 8.
1670///
1671/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm256_i64gather_epi64)
1672#[inline]
1673#[target_feature(enable = "avx2")]
1674#[cfg_attr(test, assert_instr(vpgatherqq, SCALE = 1))]
1675#[rustc_legacy_const_generics(2)]
1676#[stable(feature = "simd_x86", since = "1.27.0")]
1677pub unsafe fn _mm256_i64gather_epi64<const SCALE: i32>(
1678    slice: *const i64,
1679    offsets: __m256i,
1680) -> __m256i {
1681    static_assert_imm8_scale!(SCALE);
1682    let zero = i64x4::ZERO;
1683    let neg_one = _mm256_set1_epi64x(-1).as_i64x4();
1684    let slice = slice as *const i8;
1685    let offsets = offsets.as_i64x4();
1686    let r = vpgatherqq(zero, slice, offsets, neg_one, SCALE as i8);
1687    transmute(r)
1688}
1689
1690/// Returns values from `slice` at offsets determined by `offsets * scale`,
1691/// where
1692/// `scale` should be 1, 2, 4 or 8. If mask is set, load the value from `src` in
1693/// that position instead.
1694///
1695/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm256_mask_i64gather_epi64)
1696#[inline]
1697#[target_feature(enable = "avx2")]
1698#[cfg_attr(test, assert_instr(vpgatherqq, SCALE = 1))]
1699#[rustc_legacy_const_generics(4)]
1700#[stable(feature = "simd_x86", since = "1.27.0")]
1701pub unsafe fn _mm256_mask_i64gather_epi64<const SCALE: i32>(
1702    src: __m256i,
1703    slice: *const i64,
1704    offsets: __m256i,
1705    mask: __m256i,
1706) -> __m256i {
1707    static_assert_imm8_scale!(SCALE);
1708    let src = src.as_i64x4();
1709    let mask = mask.as_i64x4();
1710    let offsets = offsets.as_i64x4();
1711    let slice = slice as *const i8;
1712    let r = vpgatherqq(src, slice, offsets, mask, SCALE as i8);
1713    transmute(r)
1714}
1715
1716/// Returns values from `slice` at offsets determined by `offsets * scale`,
1717/// where
1718/// `scale` should be 1, 2, 4 or 8.
1719///
1720/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_i64gather_pd)
1721#[inline]
1722#[target_feature(enable = "avx2")]
1723#[cfg_attr(test, assert_instr(vgatherqpd, SCALE = 1))]
1724#[rustc_legacy_const_generics(2)]
1725#[stable(feature = "simd_x86", since = "1.27.0")]
1726pub unsafe fn _mm_i64gather_pd<const SCALE: i32>(slice: *const f64, offsets: __m128i) -> __m128d {
1727    static_assert_imm8_scale!(SCALE);
1728    let zero = _mm_setzero_pd();
1729    let neg_one = _mm_set1_pd(-1.0);
1730    let slice = slice as *const i8;
1731    let offsets = offsets.as_i64x2();
1732    pgatherqpd(zero, slice, offsets, neg_one, SCALE as i8)
1733}
1734
1735/// Returns values from `slice` at offsets determined by `offsets * scale`,
1736/// where
1737/// `scale` should be 1, 2, 4 or 8. If mask is set, load the value from `src` in
1738/// that position instead.
1739///
1740/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_mask_i64gather_pd)
1741#[inline]
1742#[target_feature(enable = "avx2")]
1743#[cfg_attr(test, assert_instr(vgatherqpd, SCALE = 1))]
1744#[rustc_legacy_const_generics(4)]
1745#[stable(feature = "simd_x86", since = "1.27.0")]
1746pub unsafe fn _mm_mask_i64gather_pd<const SCALE: i32>(
1747    src: __m128d,
1748    slice: *const f64,
1749    offsets: __m128i,
1750    mask: __m128d,
1751) -> __m128d {
1752    static_assert_imm8_scale!(SCALE);
1753    let slice = slice as *const i8;
1754    let offsets = offsets.as_i64x2();
1755    pgatherqpd(src, slice, offsets, mask, SCALE as i8)
1756}
1757
1758/// Returns values from `slice` at offsets determined by `offsets * scale`,
1759/// where
1760/// `scale` should be 1, 2, 4 or 8.
1761///
1762/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm256_i64gather_pd)
1763#[inline]
1764#[target_feature(enable = "avx2")]
1765#[cfg_attr(test, assert_instr(vgatherqpd, SCALE = 1))]
1766#[rustc_legacy_const_generics(2)]
1767#[stable(feature = "simd_x86", since = "1.27.0")]
1768pub unsafe fn _mm256_i64gather_pd<const SCALE: i32>(
1769    slice: *const f64,
1770    offsets: __m256i,
1771) -> __m256d {
1772    static_assert_imm8_scale!(SCALE);
1773    let zero = _mm256_setzero_pd();
1774    let neg_one = _mm256_set1_pd(-1.0);
1775    let slice = slice as *const i8;
1776    let offsets = offsets.as_i64x4();
1777    vpgatherqpd(zero, slice, offsets, neg_one, SCALE as i8)
1778}
1779
1780/// Returns values from `slice` at offsets determined by `offsets * scale`,
1781/// where
1782/// `scale` should be 1, 2, 4 or 8. If mask is set, load the value from `src` in
1783/// that position instead.
1784///
1785/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm256_mask_i64gather_pd)
1786#[inline]
1787#[target_feature(enable = "avx2")]
1788#[cfg_attr(test, assert_instr(vgatherqpd, SCALE = 1))]
1789#[rustc_legacy_const_generics(4)]
1790#[stable(feature = "simd_x86", since = "1.27.0")]
1791pub unsafe fn _mm256_mask_i64gather_pd<const SCALE: i32>(
1792    src: __m256d,
1793    slice: *const f64,
1794    offsets: __m256i,
1795    mask: __m256d,
1796) -> __m256d {
1797    static_assert_imm8_scale!(SCALE);
1798    let slice = slice as *const i8;
1799    let offsets = offsets.as_i64x4();
1800    vpgatherqpd(src, slice, offsets, mask, SCALE as i8)
1801}
1802
1803/// Copies `a` to `dst`, then insert 128 bits (of integer data) from `b` at the
1804/// location specified by `IMM1`.
1805///
1806/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm256_inserti128_si256)
1807#[inline]
1808#[target_feature(enable = "avx2")]
1809#[cfg_attr(test, assert_instr(vinsertf128, IMM1 = 1))]
1810#[rustc_legacy_const_generics(2)]
1811#[stable(feature = "simd_x86", since = "1.27.0")]
1812#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")]
1813pub const fn _mm256_inserti128_si256<const IMM1: i32>(a: __m256i, b: __m128i) -> __m256i {
1814    static_assert_uimm_bits!(IMM1, 1);
1815    unsafe {
1816        let a = a.as_i64x4();
1817        let b = _mm256_castsi128_si256(b).as_i64x4();
1818        let dst: i64x4 = simd_shuffle!(a, b, [[4, 5, 2, 3], [0, 1, 4, 5]][IMM1 as usize]);
1819        transmute(dst)
1820    }
1821}
1822
1823/// Multiplies packed signed 16-bit integers in `a` and `b`, producing
1824/// intermediate signed 32-bit integers. Horizontally add adjacent pairs
1825/// of intermediate 32-bit integers.
1826///
1827/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm256_madd_epi16)
1828#[inline]
1829#[target_feature(enable = "avx2")]
1830#[cfg_attr(test, assert_instr(vpmaddwd))]
1831#[stable(feature = "simd_x86", since = "1.27.0")]
1832pub fn _mm256_madd_epi16(a: __m256i, b: __m256i) -> __m256i {
1833    // It's a trick used in the Adler-32 algorithm to perform a widening addition.
1834    //
1835    // ```rust
1836    // #[target_feature(enable = "avx2")]
1837    // unsafe fn widening_add(mad: __m256i) -> __m256i {
1838    //     _mm256_madd_epi16(mad, _mm256_set1_epi16(1))
1839    // }
1840    // ```
1841    //
1842    // If we implement this using generic vector intrinsics, the optimizer
1843    // will eliminate this pattern, and `vpmaddwd` will no longer be emitted.
1844    // For this reason, we use x86 intrinsics.
1845    unsafe { transmute(pmaddwd(a.as_i16x16(), b.as_i16x16())) }
1846}
1847
1848/// Vertically multiplies each unsigned 8-bit integer from `a` with the
1849/// corresponding signed 8-bit integer from `b`, producing intermediate
1850/// signed 16-bit integers. Horizontally add adjacent pairs of intermediate
1851/// signed 16-bit integers
1852///
1853/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm256_maddubs_epi16)
1854#[inline]
1855#[target_feature(enable = "avx2")]
1856#[cfg_attr(test, assert_instr(vpmaddubsw))]
1857#[stable(feature = "simd_x86", since = "1.27.0")]
1858pub fn _mm256_maddubs_epi16(a: __m256i, b: __m256i) -> __m256i {
1859    unsafe { transmute(pmaddubsw(a.as_u8x32(), b.as_i8x32())) }
1860}
1861
1862/// Loads packed 32-bit integers from memory pointed by `mem_addr` using `mask`
1863/// (elements are zeroed out when the highest bit is not set in the
1864/// corresponding element).
1865///
1866/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_maskload_epi32)
1867#[inline]
1868#[target_feature(enable = "avx2")]
1869#[cfg_attr(test, assert_instr(vpmaskmovd))]
1870#[stable(feature = "simd_x86", since = "1.27.0")]
1871#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")]
1872pub const unsafe fn _mm_maskload_epi32(mem_addr: *const i32, mask: __m128i) -> __m128i {
1873    let mask = simd_shr(mask.as_i32x4(), i32x4::splat(31));
1874    simd_masked_load!(SimdAlign::Unaligned, mask, mem_addr, i32x4::ZERO).as_m128i()
1875}
1876
1877/// Loads packed 32-bit integers from memory pointed by `mem_addr` using `mask`
1878/// (elements are zeroed out when the highest bit is not set in the
1879/// corresponding element).
1880///
1881/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm256_maskload_epi32)
1882#[inline]
1883#[target_feature(enable = "avx2")]
1884#[cfg_attr(test, assert_instr(vpmaskmovd))]
1885#[stable(feature = "simd_x86", since = "1.27.0")]
1886#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")]
1887pub const unsafe fn _mm256_maskload_epi32(mem_addr: *const i32, mask: __m256i) -> __m256i {
1888    let mask = simd_shr(mask.as_i32x8(), i32x8::splat(31));
1889    simd_masked_load!(SimdAlign::Unaligned, mask, mem_addr, i32x8::ZERO).as_m256i()
1890}
1891
1892/// Loads packed 64-bit integers from memory pointed by `mem_addr` using `mask`
1893/// (elements are zeroed out when the highest bit is not set in the
1894/// corresponding element).
1895///
1896/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_maskload_epi64)
1897#[inline]
1898#[target_feature(enable = "avx2")]
1899#[cfg_attr(test, assert_instr(vpmaskmovq))]
1900#[stable(feature = "simd_x86", since = "1.27.0")]
1901#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")]
1902pub const unsafe fn _mm_maskload_epi64(mem_addr: *const i64, mask: __m128i) -> __m128i {
1903    let mask = simd_shr(mask.as_i64x2(), i64x2::splat(63));
1904    simd_masked_load!(SimdAlign::Unaligned, mask, mem_addr, i64x2::ZERO).as_m128i()
1905}
1906
1907/// Loads packed 64-bit integers from memory pointed by `mem_addr` using `mask`
1908/// (elements are zeroed out when the highest bit is not set in the
1909/// corresponding element).
1910///
1911/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm256_maskload_epi64)
1912#[inline]
1913#[target_feature(enable = "avx2")]
1914#[cfg_attr(test, assert_instr(vpmaskmovq))]
1915#[stable(feature = "simd_x86", since = "1.27.0")]
1916#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")]
1917pub const unsafe fn _mm256_maskload_epi64(mem_addr: *const i64, mask: __m256i) -> __m256i {
1918    let mask = simd_shr(mask.as_i64x4(), i64x4::splat(63));
1919    simd_masked_load!(SimdAlign::Unaligned, mask, mem_addr, i64x4::ZERO).as_m256i()
1920}
1921
1922/// Stores packed 32-bit integers from `a` into memory pointed by `mem_addr`
1923/// using `mask` (elements are not stored when the highest bit is not set
1924/// in the corresponding element).
1925///
1926/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_maskstore_epi32)
1927#[inline]
1928#[target_feature(enable = "avx2")]
1929#[cfg_attr(test, assert_instr(vpmaskmovd))]
1930#[stable(feature = "simd_x86", since = "1.27.0")]
1931#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")]
1932pub const unsafe fn _mm_maskstore_epi32(mem_addr: *mut i32, mask: __m128i, a: __m128i) {
1933    let mask = simd_shr(mask.as_i32x4(), i32x4::splat(31));
1934    simd_masked_store!(SimdAlign::Unaligned, mask, mem_addr, a.as_i32x4())
1935}
1936
1937/// Stores packed 32-bit integers from `a` into memory pointed by `mem_addr`
1938/// using `mask` (elements are not stored when the highest bit is not set
1939/// in the corresponding element).
1940///
1941/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm256_maskstore_epi32)
1942#[inline]
1943#[target_feature(enable = "avx2")]
1944#[cfg_attr(test, assert_instr(vpmaskmovd))]
1945#[stable(feature = "simd_x86", since = "1.27.0")]
1946#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")]
1947pub const unsafe fn _mm256_maskstore_epi32(mem_addr: *mut i32, mask: __m256i, a: __m256i) {
1948    let mask = simd_shr(mask.as_i32x8(), i32x8::splat(31));
1949    simd_masked_store!(SimdAlign::Unaligned, mask, mem_addr, a.as_i32x8())
1950}
1951
1952/// Stores packed 64-bit integers from `a` into memory pointed by `mem_addr`
1953/// using `mask` (elements are not stored when the highest bit is not set
1954/// in the corresponding element).
1955///
1956/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_maskstore_epi64)
1957#[inline]
1958#[target_feature(enable = "avx2")]
1959#[cfg_attr(test, assert_instr(vpmaskmovq))]
1960#[stable(feature = "simd_x86", since = "1.27.0")]
1961#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")]
1962pub const unsafe fn _mm_maskstore_epi64(mem_addr: *mut i64, mask: __m128i, a: __m128i) {
1963    let mask = simd_shr(mask.as_i64x2(), i64x2::splat(63));
1964    simd_masked_store!(SimdAlign::Unaligned, mask, mem_addr, a.as_i64x2())
1965}
1966
1967/// Stores packed 64-bit integers from `a` into memory pointed by `mem_addr`
1968/// using `mask` (elements are not stored when the highest bit is not set
1969/// in the corresponding element).
1970///
1971/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm256_maskstore_epi64)
1972#[inline]
1973#[target_feature(enable = "avx2")]
1974#[cfg_attr(test, assert_instr(vpmaskmovq))]
1975#[stable(feature = "simd_x86", since = "1.27.0")]
1976#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")]
1977pub const unsafe fn _mm256_maskstore_epi64(mem_addr: *mut i64, mask: __m256i, a: __m256i) {
1978    let mask = simd_shr(mask.as_i64x4(), i64x4::splat(63));
1979    simd_masked_store!(SimdAlign::Unaligned, mask, mem_addr, a.as_i64x4())
1980}
1981
1982/// Compares packed 16-bit integers in `a` and `b`, and returns the packed
1983/// maximum values.
1984///
1985/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm256_max_epi16)
1986#[inline]
1987#[target_feature(enable = "avx2")]
1988#[cfg_attr(test, assert_instr(vpmaxsw))]
1989#[stable(feature = "simd_x86", since = "1.27.0")]
1990#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")]
1991pub const fn _mm256_max_epi16(a: __m256i, b: __m256i) -> __m256i {
1992    unsafe { simd_imax(a.as_i16x16(), b.as_i16x16()).as_m256i() }
1993}
1994
1995/// Compares packed 32-bit integers in `a` and `b`, and returns the packed
1996/// maximum values.
1997///
1998/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm256_max_epi32)
1999#[inline]
2000#[target_feature(enable = "avx2")]
2001#[cfg_attr(test, assert_instr(vpmaxsd))]
2002#[stable(feature = "simd_x86", since = "1.27.0")]
2003#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")]
2004pub const fn _mm256_max_epi32(a: __m256i, b: __m256i) -> __m256i {
2005    unsafe { simd_imax(a.as_i32x8(), b.as_i32x8()).as_m256i() }
2006}
2007
2008/// Compares packed 8-bit integers in `a` and `b`, and returns the packed
2009/// maximum values.
2010///
2011/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm256_max_epi8)
2012#[inline]
2013#[target_feature(enable = "avx2")]
2014#[cfg_attr(test, assert_instr(vpmaxsb))]
2015#[stable(feature = "simd_x86", since = "1.27.0")]
2016#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")]
2017pub const fn _mm256_max_epi8(a: __m256i, b: __m256i) -> __m256i {
2018    unsafe { simd_imax(a.as_i8x32(), b.as_i8x32()).as_m256i() }
2019}
2020
2021/// Compares packed unsigned 16-bit integers in `a` and `b`, and returns
2022/// the packed maximum values.
2023///
2024/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm256_max_epu16)
2025#[inline]
2026#[target_feature(enable = "avx2")]
2027#[cfg_attr(test, assert_instr(vpmaxuw))]
2028#[stable(feature = "simd_x86", since = "1.27.0")]
2029#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")]
2030pub const fn _mm256_max_epu16(a: __m256i, b: __m256i) -> __m256i {
2031    unsafe { simd_imax(a.as_u16x16(), b.as_u16x16()).as_m256i() }
2032}
2033
2034/// Compares packed unsigned 32-bit integers in `a` and `b`, and returns
2035/// the packed maximum values.
2036///
2037/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm256_max_epu32)
2038#[inline]
2039#[target_feature(enable = "avx2")]
2040#[cfg_attr(test, assert_instr(vpmaxud))]
2041#[stable(feature = "simd_x86", since = "1.27.0")]
2042#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")]
2043pub const fn _mm256_max_epu32(a: __m256i, b: __m256i) -> __m256i {
2044    unsafe { simd_imax(a.as_u32x8(), b.as_u32x8()).as_m256i() }
2045}
2046
2047/// Compares packed unsigned 8-bit integers in `a` and `b`, and returns
2048/// the packed maximum values.
2049///
2050/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm256_max_epu8)
2051#[inline]
2052#[target_feature(enable = "avx2")]
2053#[cfg_attr(test, assert_instr(vpmaxub))]
2054#[stable(feature = "simd_x86", since = "1.27.0")]
2055#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")]
2056pub const fn _mm256_max_epu8(a: __m256i, b: __m256i) -> __m256i {
2057    unsafe { simd_imax(a.as_u8x32(), b.as_u8x32()).as_m256i() }
2058}
2059
2060/// Compares packed 16-bit integers in `a` and `b`, and returns the packed
2061/// minimum values.
2062///
2063/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm256_min_epi16)
2064#[inline]
2065#[target_feature(enable = "avx2")]
2066#[cfg_attr(test, assert_instr(vpminsw))]
2067#[stable(feature = "simd_x86", since = "1.27.0")]
2068#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")]
2069pub const fn _mm256_min_epi16(a: __m256i, b: __m256i) -> __m256i {
2070    unsafe { simd_imin(a.as_i16x16(), b.as_i16x16()).as_m256i() }
2071}
2072
2073/// Compares packed 32-bit integers in `a` and `b`, and returns the packed
2074/// minimum values.
2075///
2076/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm256_min_epi32)
2077#[inline]
2078#[target_feature(enable = "avx2")]
2079#[cfg_attr(test, assert_instr(vpminsd))]
2080#[stable(feature = "simd_x86", since = "1.27.0")]
2081#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")]
2082pub const fn _mm256_min_epi32(a: __m256i, b: __m256i) -> __m256i {
2083    unsafe { simd_imin(a.as_i32x8(), b.as_i32x8()).as_m256i() }
2084}
2085
2086/// Compares packed 8-bit integers in `a` and `b`, and returns the packed
2087/// minimum values.
2088///
2089/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm256_min_epi8)
2090#[inline]
2091#[target_feature(enable = "avx2")]
2092#[cfg_attr(test, assert_instr(vpminsb))]
2093#[stable(feature = "simd_x86", since = "1.27.0")]
2094#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")]
2095pub const fn _mm256_min_epi8(a: __m256i, b: __m256i) -> __m256i {
2096    unsafe { simd_imin(a.as_i8x32(), b.as_i8x32()).as_m256i() }
2097}
2098
2099/// Compares packed unsigned 16-bit integers in `a` and `b`, and returns
2100/// the packed minimum values.
2101///
2102/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm256_min_epu16)
2103#[inline]
2104#[target_feature(enable = "avx2")]
2105#[cfg_attr(test, assert_instr(vpminuw))]
2106#[stable(feature = "simd_x86", since = "1.27.0")]
2107#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")]
2108pub const fn _mm256_min_epu16(a: __m256i, b: __m256i) -> __m256i {
2109    unsafe { simd_imin(a.as_u16x16(), b.as_u16x16()).as_m256i() }
2110}
2111
2112/// Compares packed unsigned 32-bit integers in `a` and `b`, and returns
2113/// the packed minimum values.
2114///
2115/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm256_min_epu32)
2116#[inline]
2117#[target_feature(enable = "avx2")]
2118#[cfg_attr(test, assert_instr(vpminud))]
2119#[stable(feature = "simd_x86", since = "1.27.0")]
2120#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")]
2121pub const fn _mm256_min_epu32(a: __m256i, b: __m256i) -> __m256i {
2122    unsafe { simd_imin(a.as_u32x8(), b.as_u32x8()).as_m256i() }
2123}
2124
2125/// Compares packed unsigned 8-bit integers in `a` and `b`, and returns
2126/// the packed minimum values.
2127///
2128/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm256_min_epu8)
2129#[inline]
2130#[target_feature(enable = "avx2")]
2131#[cfg_attr(test, assert_instr(vpminub))]
2132#[stable(feature = "simd_x86", since = "1.27.0")]
2133#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")]
2134pub const fn _mm256_min_epu8(a: __m256i, b: __m256i) -> __m256i {
2135    unsafe { simd_imin(a.as_u8x32(), b.as_u8x32()).as_m256i() }
2136}
2137
2138/// Creates mask from the most significant bit of each 8-bit element in `a`,
2139/// return the result.
2140///
2141/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm256_movemask_epi8)
2142#[inline]
2143#[target_feature(enable = "avx2")]
2144#[cfg_attr(test, assert_instr(vpmovmskb))]
2145#[stable(feature = "simd_x86", since = "1.27.0")]
2146#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")]
2147pub const fn _mm256_movemask_epi8(a: __m256i) -> i32 {
2148    unsafe {
2149        let z = i8x32::ZERO;
2150        let m: i8x32 = simd_lt(a.as_i8x32(), z);
2151        simd_bitmask::<_, u32>(m) as i32
2152    }
2153}
2154
2155/// Computes the sum of absolute differences (SADs) of quadruplets of unsigned
2156/// 8-bit integers in `a` compared to those in `b`, and stores the 16-bit
2157/// results in dst. Eight SADs are performed for each 128-bit lane using one
2158/// quadruplet from `b` and eight quadruplets from `a`. One quadruplet is
2159/// selected from `b` starting at on the offset specified in `imm8`. Eight
2160/// quadruplets are formed from sequential 8-bit integers selected from `a`
2161/// starting at the offset specified in `imm8`.
2162///
2163/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm256_mpsadbw_epu8)
2164#[inline]
2165#[target_feature(enable = "avx2")]
2166#[cfg_attr(test, assert_instr(vmpsadbw, IMM8 = 0))]
2167#[rustc_legacy_const_generics(2)]
2168#[stable(feature = "simd_x86", since = "1.27.0")]
2169pub fn _mm256_mpsadbw_epu8<const IMM8: i32>(a: __m256i, b: __m256i) -> __m256i {
2170    static_assert_uimm_bits!(IMM8, 8);
2171    unsafe { transmute(mpsadbw(a.as_u8x32(), b.as_u8x32(), IMM8 as i8)) }
2172}
2173
2174/// Multiplies the low 32-bit integers from each packed 64-bit element in
2175/// `a` and `b`
2176///
2177/// Returns the 64-bit results.
2178///
2179/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm256_mul_epi32)
2180#[inline]
2181#[target_feature(enable = "avx2")]
2182#[cfg_attr(test, assert_instr(vpmuldq))]
2183#[stable(feature = "simd_x86", since = "1.27.0")]
2184#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")]
2185pub const fn _mm256_mul_epi32(a: __m256i, b: __m256i) -> __m256i {
2186    unsafe {
2187        let a = simd_cast::<_, i64x4>(simd_cast::<_, i32x4>(a.as_i64x4()));
2188        let b = simd_cast::<_, i64x4>(simd_cast::<_, i32x4>(b.as_i64x4()));
2189        transmute(simd_mul(a, b))
2190    }
2191}
2192
2193/// Multiplies the low unsigned 32-bit integers from each packed 64-bit
2194/// element in `a` and `b`
2195///
2196/// Returns the unsigned 64-bit results.
2197///
2198/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm256_mul_epu32)
2199#[inline]
2200#[target_feature(enable = "avx2")]
2201#[cfg_attr(test, assert_instr(vpmuludq))]
2202#[stable(feature = "simd_x86", since = "1.27.0")]
2203#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")]
2204pub const fn _mm256_mul_epu32(a: __m256i, b: __m256i) -> __m256i {
2205    unsafe {
2206        let a = a.as_u64x4();
2207        let b = b.as_u64x4();
2208        let mask = u64x4::splat(u32::MAX as u64);
2209        transmute(simd_mul(simd_and(a, mask), simd_and(b, mask)))
2210    }
2211}
2212
2213/// Multiplies the packed 16-bit integers in `a` and `b`, producing
2214/// intermediate 32-bit integers and returning the high 16 bits of the
2215/// intermediate integers.
2216///
2217/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm256_mulhi_epi16)
2218#[inline]
2219#[target_feature(enable = "avx2")]
2220#[cfg_attr(test, assert_instr(vpmulhw))]
2221#[stable(feature = "simd_x86", since = "1.27.0")]
2222pub fn _mm256_mulhi_epi16(a: __m256i, b: __m256i) -> __m256i {
2223    unsafe { transmute(pmulhw(a.as_i16x16(), b.as_i16x16())) }
2224}
2225
2226/// Multiplies the packed unsigned 16-bit integers in `a` and `b`, producing
2227/// intermediate 32-bit integers and returning the high 16 bits of the
2228/// intermediate integers.
2229///
2230/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm256_mulhi_epu16)
2231#[inline]
2232#[target_feature(enable = "avx2")]
2233#[cfg_attr(test, assert_instr(vpmulhuw))]
2234#[stable(feature = "simd_x86", since = "1.27.0")]
2235pub fn _mm256_mulhi_epu16(a: __m256i, b: __m256i) -> __m256i {
2236    unsafe { transmute(pmulhuw(a.as_u16x16(), b.as_u16x16())) }
2237}
2238
2239/// Multiplies the packed 16-bit integers in `a` and `b`, producing
2240/// intermediate 32-bit integers, and returns the low 16 bits of the
2241/// intermediate integers
2242///
2243/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm256_mullo_epi16)
2244#[inline]
2245#[target_feature(enable = "avx2")]
2246#[cfg_attr(test, assert_instr(vpmullw))]
2247#[stable(feature = "simd_x86", since = "1.27.0")]
2248#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")]
2249pub const fn _mm256_mullo_epi16(a: __m256i, b: __m256i) -> __m256i {
2250    unsafe { transmute(simd_mul(a.as_i16x16(), b.as_i16x16())) }
2251}
2252
2253/// Multiplies the packed 32-bit integers in `a` and `b`, producing
2254/// intermediate 64-bit integers, and returns the low 32 bits of the
2255/// intermediate integers
2256///
2257/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm256_mullo_epi32)
2258#[inline]
2259#[target_feature(enable = "avx2")]
2260#[cfg_attr(test, assert_instr(vpmulld))]
2261#[stable(feature = "simd_x86", since = "1.27.0")]
2262#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")]
2263pub const fn _mm256_mullo_epi32(a: __m256i, b: __m256i) -> __m256i {
2264    unsafe { transmute(simd_mul(a.as_i32x8(), b.as_i32x8())) }
2265}
2266
2267/// Multiplies packed 16-bit integers in `a` and `b`, producing
2268/// intermediate signed 32-bit integers. Truncate each intermediate
2269/// integer to the 18 most significant bits, round by adding 1, and
2270/// return bits `[16:1]`.
2271///
2272/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm256_mulhrs_epi16)
2273#[inline]
2274#[target_feature(enable = "avx2")]
2275#[cfg_attr(test, assert_instr(vpmulhrsw))]
2276#[stable(feature = "simd_x86", since = "1.27.0")]
2277pub fn _mm256_mulhrs_epi16(a: __m256i, b: __m256i) -> __m256i {
2278    unsafe { transmute(pmulhrsw(a.as_i16x16(), b.as_i16x16())) }
2279}
2280
2281/// Computes the bitwise OR of 256 bits (representing integer data) in `a`
2282/// and `b`
2283///
2284/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm256_or_si256)
2285#[inline]
2286#[target_feature(enable = "avx2")]
2287#[cfg_attr(test, assert_instr(vorps))]
2288#[stable(feature = "simd_x86", since = "1.27.0")]
2289#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")]
2290pub const fn _mm256_or_si256(a: __m256i, b: __m256i) -> __m256i {
2291    unsafe { transmute(simd_or(a.as_i32x8(), b.as_i32x8())) }
2292}
2293
2294/// Converts packed 16-bit integers from `a` and `b` to packed 8-bit integers
2295/// using signed saturation
2296///
2297/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm256_packs_epi16)
2298#[inline]
2299#[target_feature(enable = "avx2")]
2300#[cfg_attr(test, assert_instr(vpacksswb))]
2301#[stable(feature = "simd_x86", since = "1.27.0")]
2302pub fn _mm256_packs_epi16(a: __m256i, b: __m256i) -> __m256i {
2303    unsafe { transmute(packsswb(a.as_i16x16(), b.as_i16x16())) }
2304}
2305
2306/// Converts packed 32-bit integers from `a` and `b` to packed 16-bit integers
2307/// using signed saturation
2308///
2309/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm256_packs_epi32)
2310#[inline]
2311#[target_feature(enable = "avx2")]
2312#[cfg_attr(test, assert_instr(vpackssdw))]
2313#[stable(feature = "simd_x86", since = "1.27.0")]
2314pub fn _mm256_packs_epi32(a: __m256i, b: __m256i) -> __m256i {
2315    unsafe { transmute(packssdw(a.as_i32x8(), b.as_i32x8())) }
2316}
2317
2318/// Converts packed 16-bit integers from `a` and `b` to packed 8-bit integers
2319/// using unsigned saturation
2320///
2321/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm256_packus_epi16)
2322#[inline]
2323#[target_feature(enable = "avx2")]
2324#[cfg_attr(test, assert_instr(vpackuswb))]
2325#[stable(feature = "simd_x86", since = "1.27.0")]
2326pub fn _mm256_packus_epi16(a: __m256i, b: __m256i) -> __m256i {
2327    unsafe { transmute(packuswb(a.as_i16x16(), b.as_i16x16())) }
2328}
2329
2330/// Converts packed 32-bit integers from `a` and `b` to packed 16-bit integers
2331/// using unsigned saturation
2332///
2333/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm256_packus_epi32)
2334#[inline]
2335#[target_feature(enable = "avx2")]
2336#[cfg_attr(test, assert_instr(vpackusdw))]
2337#[stable(feature = "simd_x86", since = "1.27.0")]
2338pub fn _mm256_packus_epi32(a: __m256i, b: __m256i) -> __m256i {
2339    unsafe { transmute(packusdw(a.as_i32x8(), b.as_i32x8())) }
2340}
2341
2342/// Permutes packed 32-bit integers from `a` according to the content of `b`.
2343///
2344/// The last 3 bits of each integer of `b` are used as addresses into the 8
2345/// integers of `a`.
2346///
2347/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm256_permutevar8x32_epi32)
2348#[inline]
2349#[target_feature(enable = "avx2")]
2350#[cfg_attr(test, assert_instr(vpermps))]
2351#[stable(feature = "simd_x86", since = "1.27.0")]
2352pub fn _mm256_permutevar8x32_epi32(a: __m256i, b: __m256i) -> __m256i {
2353    unsafe { transmute(permd(a.as_u32x8(), b.as_u32x8())) }
2354}
2355
2356/// Permutes 64-bit integers from `a` using control mask `imm8`.
2357///
2358/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm256_permute4x64_epi64)
2359#[inline]
2360#[target_feature(enable = "avx2")]
2361#[cfg_attr(test, assert_instr(vpermpd, IMM8 = 9))]
2362#[rustc_legacy_const_generics(1)]
2363#[stable(feature = "simd_x86", since = "1.27.0")]
2364#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")]
2365pub const fn _mm256_permute4x64_epi64<const IMM8: i32>(a: __m256i) -> __m256i {
2366    static_assert_uimm_bits!(IMM8, 8);
2367    unsafe {
2368        let zero = i64x4::ZERO;
2369        let r: i64x4 = simd_shuffle!(
2370            a.as_i64x4(),
2371            zero,
2372            [
2373                IMM8 as u32 & 0b11,
2374                (IMM8 as u32 >> 2) & 0b11,
2375                (IMM8 as u32 >> 4) & 0b11,
2376                (IMM8 as u32 >> 6) & 0b11,
2377            ],
2378        );
2379        transmute(r)
2380    }
2381}
2382
2383/// Shuffles 128-bits of integer data selected by `imm8` from `a` and `b`.
2384///
2385/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm256_permute2x128_si256)
2386#[inline]
2387#[target_feature(enable = "avx2")]
2388#[cfg_attr(test, assert_instr(vperm2f128, IMM8 = 9))]
2389#[rustc_legacy_const_generics(2)]
2390#[stable(feature = "simd_x86", since = "1.27.0")]
2391#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")]
2392pub const fn _mm256_permute2x128_si256<const IMM8: i32>(a: __m256i, b: __m256i) -> __m256i {
2393    static_assert_uimm_bits!(IMM8, 8);
2394    _mm256_permute2f128_si256::<IMM8>(a, b)
2395}
2396
2397/// Shuffles 64-bit floating-point elements in `a` across lanes using the
2398/// control in `imm8`.
2399///
2400/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm256_permute4x64_pd)
2401#[inline]
2402#[target_feature(enable = "avx2")]
2403#[cfg_attr(test, assert_instr(vpermpd, IMM8 = 1))]
2404#[rustc_legacy_const_generics(1)]
2405#[stable(feature = "simd_x86", since = "1.27.0")]
2406#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")]
2407pub const fn _mm256_permute4x64_pd<const IMM8: i32>(a: __m256d) -> __m256d {
2408    static_assert_uimm_bits!(IMM8, 8);
2409    unsafe {
2410        simd_shuffle!(
2411            a,
2412            _mm256_undefined_pd(),
2413            [
2414                IMM8 as u32 & 0b11,
2415                (IMM8 as u32 >> 2) & 0b11,
2416                (IMM8 as u32 >> 4) & 0b11,
2417                (IMM8 as u32 >> 6) & 0b11,
2418            ],
2419        )
2420    }
2421}
2422
2423/// Shuffles eight 32-bit floating-point elements in `a` across lanes using
2424/// the corresponding 32-bit integer index in `idx`.
2425///
2426/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm256_permutevar8x32_ps)
2427#[inline]
2428#[target_feature(enable = "avx2")]
2429#[cfg_attr(test, assert_instr(vpermps))]
2430#[stable(feature = "simd_x86", since = "1.27.0")]
2431pub fn _mm256_permutevar8x32_ps(a: __m256, idx: __m256i) -> __m256 {
2432    unsafe { permps(a, idx.as_i32x8()) }
2433}
2434
2435/// Computes the absolute differences of packed unsigned 8-bit integers in `a`
2436/// and `b`, then horizontally sum each consecutive 8 differences to
2437/// produce four unsigned 16-bit integers, and pack these unsigned 16-bit
2438/// integers in the low 16 bits of the 64-bit return value
2439///
2440/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm256_sad_epu8)
2441#[inline]
2442#[target_feature(enable = "avx2")]
2443#[cfg_attr(test, assert_instr(vpsadbw))]
2444#[stable(feature = "simd_x86", since = "1.27.0")]
2445pub fn _mm256_sad_epu8(a: __m256i, b: __m256i) -> __m256i {
2446    unsafe { transmute(psadbw(a.as_u8x32(), b.as_u8x32())) }
2447}
2448
2449/// Shuffles bytes from `a` according to the content of `b`.
2450///
2451/// For each of the 128-bit low and high halves of the vectors, the last
2452/// 4 bits of each byte of `b` are used as addresses into the respective
2453/// low or high 16 bytes of `a`. That is, the halves are shuffled separately.
2454///
2455/// In addition, if the highest significant bit of a byte of `b` is set, the
2456/// respective destination byte is set to 0.
2457///
2458/// Picturing `a` and `b` as `[u8; 32]`, `_mm256_shuffle_epi8` is logically
2459/// equivalent to:
2460///
2461/// ```
2462/// fn mm256_shuffle_epi8(a: [u8; 32], b: [u8; 32]) -> [u8; 32] {
2463///     let mut r = [0; 32];
2464///     for i in 0..16 {
2465///         // if the most significant bit of b is set,
2466///         // then the destination byte is set to 0.
2467///         if b[i] & 0x80 == 0u8 {
2468///             r[i] = a[(b[i] % 16) as usize];
2469///         }
2470///         if b[i + 16] & 0x80 == 0u8 {
2471///             r[i + 16] = a[(b[i + 16] % 16 + 16) as usize];
2472///         }
2473///     }
2474///     r
2475/// }
2476/// ```
2477///
2478/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm256_shuffle_epi8)
2479#[inline]
2480#[target_feature(enable = "avx2")]
2481#[cfg_attr(test, assert_instr(vpshufb))]
2482#[stable(feature = "simd_x86", since = "1.27.0")]
2483pub fn _mm256_shuffle_epi8(a: __m256i, b: __m256i) -> __m256i {
2484    unsafe { transmute(pshufb(a.as_u8x32(), b.as_u8x32())) }
2485}
2486
2487/// Shuffles 32-bit integers in 128-bit lanes of `a` using the control in
2488/// `imm8`.
2489///
2490/// ```rust
2491/// #[cfg(target_arch = "x86")]
2492/// use std::arch::x86::*;
2493/// #[cfg(target_arch = "x86_64")]
2494/// use std::arch::x86_64::*;
2495///
2496/// # fn main() {
2497/// #     if is_x86_feature_detected!("avx2") {
2498/// #         #[target_feature(enable = "avx2")]
2499/// #         unsafe fn worker() {
2500/// let a = _mm256_setr_epi32(0, 1, 2, 3, 4, 5, 6, 7);
2501///
2502/// let c1 = _mm256_shuffle_epi32(a, 0b00_11_10_01);
2503/// let c2 = _mm256_shuffle_epi32(a, 0b01_00_10_11);
2504///
2505/// let expected1 = _mm256_setr_epi32(1, 2, 3, 0, 5, 6, 7, 4);
2506/// let expected2 = _mm256_setr_epi32(3, 2, 0, 1, 7, 6, 4, 5);
2507///
2508/// assert_eq!(_mm256_movemask_epi8(_mm256_cmpeq_epi8(c1, expected1)), !0);
2509/// assert_eq!(_mm256_movemask_epi8(_mm256_cmpeq_epi8(c2, expected2)), !0);
2510/// #         }
2511/// #         unsafe { worker(); }
2512/// #     }
2513/// # }
2514/// ```
2515///
2516/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm256_shuffle_epi32)
2517#[inline]
2518#[target_feature(enable = "avx2")]
2519#[cfg_attr(test, assert_instr(vshufps, MASK = 9))]
2520#[rustc_legacy_const_generics(1)]
2521#[stable(feature = "simd_x86", since = "1.27.0")]
2522#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")]
2523pub const fn _mm256_shuffle_epi32<const MASK: i32>(a: __m256i) -> __m256i {
2524    static_assert_uimm_bits!(MASK, 8);
2525    unsafe {
2526        let r: i32x8 = simd_shuffle!(
2527            a.as_i32x8(),
2528            a.as_i32x8(),
2529            [
2530                MASK as u32 & 0b11,
2531                (MASK as u32 >> 2) & 0b11,
2532                (MASK as u32 >> 4) & 0b11,
2533                (MASK as u32 >> 6) & 0b11,
2534                (MASK as u32 & 0b11) + 4,
2535                ((MASK as u32 >> 2) & 0b11) + 4,
2536                ((MASK as u32 >> 4) & 0b11) + 4,
2537                ((MASK as u32 >> 6) & 0b11) + 4,
2538            ],
2539        );
2540        transmute(r)
2541    }
2542}
2543
2544/// Shuffles 16-bit integers in the high 64 bits of 128-bit lanes of `a` using
2545/// the control in `imm8`. The low 64 bits of 128-bit lanes of `a` are copied
2546/// to the output.
2547///
2548/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm256_shufflehi_epi16)
2549#[inline]
2550#[target_feature(enable = "avx2")]
2551#[cfg_attr(test, assert_instr(vpshufhw, IMM8 = 9))]
2552#[rustc_legacy_const_generics(1)]
2553#[stable(feature = "simd_x86", since = "1.27.0")]
2554#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")]
2555pub const fn _mm256_shufflehi_epi16<const IMM8: i32>(a: __m256i) -> __m256i {
2556    static_assert_uimm_bits!(IMM8, 8);
2557    unsafe {
2558        let a = a.as_i16x16();
2559        let r: i16x16 = simd_shuffle!(
2560            a,
2561            a,
2562            [
2563                0,
2564                1,
2565                2,
2566                3,
2567                4 + (IMM8 as u32 & 0b11),
2568                4 + ((IMM8 as u32 >> 2) & 0b11),
2569                4 + ((IMM8 as u32 >> 4) & 0b11),
2570                4 + ((IMM8 as u32 >> 6) & 0b11),
2571                8,
2572                9,
2573                10,
2574                11,
2575                12 + (IMM8 as u32 & 0b11),
2576                12 + ((IMM8 as u32 >> 2) & 0b11),
2577                12 + ((IMM8 as u32 >> 4) & 0b11),
2578                12 + ((IMM8 as u32 >> 6) & 0b11),
2579            ],
2580        );
2581        transmute(r)
2582    }
2583}
2584
2585/// Shuffles 16-bit integers in the low 64 bits of 128-bit lanes of `a` using
2586/// the control in `imm8`. The high 64 bits of 128-bit lanes of `a` are copied
2587/// to the output.
2588///
2589/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm256_shufflelo_epi16)
2590#[inline]
2591#[target_feature(enable = "avx2")]
2592#[cfg_attr(test, assert_instr(vpshuflw, IMM8 = 9))]
2593#[rustc_legacy_const_generics(1)]
2594#[stable(feature = "simd_x86", since = "1.27.0")]
2595#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")]
2596pub const fn _mm256_shufflelo_epi16<const IMM8: i32>(a: __m256i) -> __m256i {
2597    static_assert_uimm_bits!(IMM8, 8);
2598    unsafe {
2599        let a = a.as_i16x16();
2600        let r: i16x16 = simd_shuffle!(
2601            a,
2602            a,
2603            [
2604                0 + (IMM8 as u32 & 0b11),
2605                0 + ((IMM8 as u32 >> 2) & 0b11),
2606                0 + ((IMM8 as u32 >> 4) & 0b11),
2607                0 + ((IMM8 as u32 >> 6) & 0b11),
2608                4,
2609                5,
2610                6,
2611                7,
2612                8 + (IMM8 as u32 & 0b11),
2613                8 + ((IMM8 as u32 >> 2) & 0b11),
2614                8 + ((IMM8 as u32 >> 4) & 0b11),
2615                8 + ((IMM8 as u32 >> 6) & 0b11),
2616                12,
2617                13,
2618                14,
2619                15,
2620            ],
2621        );
2622        transmute(r)
2623    }
2624}
2625
2626/// Negates packed 16-bit integers in `a` when the corresponding signed
2627/// 16-bit integer in `b` is negative, and returns the results.
2628/// Results are zeroed out when the corresponding element in `b` is zero.
2629///
2630/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm256_sign_epi16)
2631#[inline]
2632#[target_feature(enable = "avx2")]
2633#[cfg_attr(test, assert_instr(vpsignw))]
2634#[stable(feature = "simd_x86", since = "1.27.0")]
2635pub fn _mm256_sign_epi16(a: __m256i, b: __m256i) -> __m256i {
2636    unsafe { transmute(psignw(a.as_i16x16(), b.as_i16x16())) }
2637}
2638
2639/// Negates packed 32-bit integers in `a` when the corresponding signed
2640/// 32-bit integer in `b` is negative, and returns the results.
2641/// Results are zeroed out when the corresponding element in `b` is zero.
2642///
2643/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm256_sign_epi32)
2644#[inline]
2645#[target_feature(enable = "avx2")]
2646#[cfg_attr(test, assert_instr(vpsignd))]
2647#[stable(feature = "simd_x86", since = "1.27.0")]
2648pub fn _mm256_sign_epi32(a: __m256i, b: __m256i) -> __m256i {
2649    unsafe { transmute(psignd(a.as_i32x8(), b.as_i32x8())) }
2650}
2651
2652/// Negates packed 8-bit integers in `a` when the corresponding signed
2653/// 8-bit integer in `b` is negative, and returns the results.
2654/// Results are zeroed out when the corresponding element in `b` is zero.
2655///
2656/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm256_sign_epi8)
2657#[inline]
2658#[target_feature(enable = "avx2")]
2659#[cfg_attr(test, assert_instr(vpsignb))]
2660#[stable(feature = "simd_x86", since = "1.27.0")]
2661pub fn _mm256_sign_epi8(a: __m256i, b: __m256i) -> __m256i {
2662    unsafe { transmute(psignb(a.as_i8x32(), b.as_i8x32())) }
2663}
2664
2665/// Shifts packed 16-bit integers in `a` left by `count` while
2666/// shifting in zeros, and returns the result
2667///
2668/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm256_sll_epi16)
2669#[inline]
2670#[target_feature(enable = "avx2")]
2671#[cfg_attr(test, assert_instr(vpsllw))]
2672#[stable(feature = "simd_x86", since = "1.27.0")]
2673pub fn _mm256_sll_epi16(a: __m256i, count: __m128i) -> __m256i {
2674    unsafe { transmute(psllw(a.as_i16x16(), count.as_i16x8())) }
2675}
2676
2677/// Shifts packed 32-bit integers in `a` left by `count` while
2678/// shifting in zeros, and returns the result
2679///
2680/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm256_sll_epi32)
2681#[inline]
2682#[target_feature(enable = "avx2")]
2683#[cfg_attr(test, assert_instr(vpslld))]
2684#[stable(feature = "simd_x86", since = "1.27.0")]
2685pub fn _mm256_sll_epi32(a: __m256i, count: __m128i) -> __m256i {
2686    unsafe { transmute(pslld(a.as_i32x8(), count.as_i32x4())) }
2687}
2688
2689/// Shifts packed 64-bit integers in `a` left by `count` while
2690/// shifting in zeros, and returns the result
2691///
2692/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm256_sll_epi64)
2693#[inline]
2694#[target_feature(enable = "avx2")]
2695#[cfg_attr(test, assert_instr(vpsllq))]
2696#[stable(feature = "simd_x86", since = "1.27.0")]
2697pub fn _mm256_sll_epi64(a: __m256i, count: __m128i) -> __m256i {
2698    unsafe { transmute(psllq(a.as_i64x4(), count.as_i64x2())) }
2699}
2700
2701/// Shifts packed 16-bit integers in `a` left by `IMM8` while
2702/// shifting in zeros, return the results;
2703///
2704/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm256_slli_epi16)
2705#[inline]
2706#[target_feature(enable = "avx2")]
2707#[cfg_attr(test, assert_instr(vpsllw, IMM8 = 7))]
2708#[rustc_legacy_const_generics(1)]
2709#[stable(feature = "simd_x86", since = "1.27.0")]
2710#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")]
2711pub const fn _mm256_slli_epi16<const IMM8: i32>(a: __m256i) -> __m256i {
2712    static_assert_uimm_bits!(IMM8, 8);
2713    unsafe {
2714        if IMM8 >= 16 {
2715            _mm256_setzero_si256()
2716        } else {
2717            transmute(simd_shl(a.as_u16x16(), u16x16::splat(IMM8 as u16)))
2718        }
2719    }
2720}
2721
2722/// Shifts packed 32-bit integers in `a` left by `IMM8` while
2723/// shifting in zeros, return the results;
2724///
2725/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm256_slli_epi32)
2726#[inline]
2727#[target_feature(enable = "avx2")]
2728#[cfg_attr(test, assert_instr(vpslld, IMM8 = 7))]
2729#[rustc_legacy_const_generics(1)]
2730#[stable(feature = "simd_x86", since = "1.27.0")]
2731#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")]
2732pub const fn _mm256_slli_epi32<const IMM8: i32>(a: __m256i) -> __m256i {
2733    unsafe {
2734        static_assert_uimm_bits!(IMM8, 8);
2735        if IMM8 >= 32 {
2736            _mm256_setzero_si256()
2737        } else {
2738            transmute(simd_shl(a.as_u32x8(), u32x8::splat(IMM8 as u32)))
2739        }
2740    }
2741}
2742
2743/// Shifts packed 64-bit integers in `a` left by `IMM8` while
2744/// shifting in zeros, return the results;
2745///
2746/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm256_slli_epi64)
2747#[inline]
2748#[target_feature(enable = "avx2")]
2749#[cfg_attr(test, assert_instr(vpsllq, IMM8 = 7))]
2750#[rustc_legacy_const_generics(1)]
2751#[stable(feature = "simd_x86", since = "1.27.0")]
2752#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")]
2753pub const fn _mm256_slli_epi64<const IMM8: i32>(a: __m256i) -> __m256i {
2754    unsafe {
2755        static_assert_uimm_bits!(IMM8, 8);
2756        if IMM8 >= 64 {
2757            _mm256_setzero_si256()
2758        } else {
2759            transmute(simd_shl(a.as_u64x4(), u64x4::splat(IMM8 as u64)))
2760        }
2761    }
2762}
2763
2764/// Shifts 128-bit lanes in `a` left by `imm8` bytes while shifting in zeros.
2765///
2766/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm256_slli_si256)
2767#[inline]
2768#[target_feature(enable = "avx2")]
2769#[cfg_attr(test, assert_instr(vpslldq, IMM8 = 3))]
2770#[rustc_legacy_const_generics(1)]
2771#[stable(feature = "simd_x86", since = "1.27.0")]
2772#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")]
2773pub const fn _mm256_slli_si256<const IMM8: i32>(a: __m256i) -> __m256i {
2774    static_assert_uimm_bits!(IMM8, 8);
2775    _mm256_bslli_epi128::<IMM8>(a)
2776}
2777
2778/// Shifts 128-bit lanes in `a` left by `imm8` bytes while shifting in zeros.
2779///
2780/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm256_bslli_epi128)
2781#[inline]
2782#[target_feature(enable = "avx2")]
2783#[cfg_attr(test, assert_instr(vpslldq, IMM8 = 3))]
2784#[rustc_legacy_const_generics(1)]
2785#[stable(feature = "simd_x86", since = "1.27.0")]
2786#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")]
2787pub const fn _mm256_bslli_epi128<const IMM8: i32>(a: __m256i) -> __m256i {
2788    static_assert_uimm_bits!(IMM8, 8);
2789    const fn mask(shift: i32, i: u32) -> u32 {
2790        let shift = shift as u32 & 0xff;
2791        if shift > 15 || i % 16 < shift {
2792            0
2793        } else {
2794            32 + (i - shift)
2795        }
2796    }
2797    unsafe {
2798        let a = a.as_i8x32();
2799        let r: i8x32 = simd_shuffle!(
2800            i8x32::ZERO,
2801            a,
2802            [
2803                mask(IMM8, 0),
2804                mask(IMM8, 1),
2805                mask(IMM8, 2),
2806                mask(IMM8, 3),
2807                mask(IMM8, 4),
2808                mask(IMM8, 5),
2809                mask(IMM8, 6),
2810                mask(IMM8, 7),
2811                mask(IMM8, 8),
2812                mask(IMM8, 9),
2813                mask(IMM8, 10),
2814                mask(IMM8, 11),
2815                mask(IMM8, 12),
2816                mask(IMM8, 13),
2817                mask(IMM8, 14),
2818                mask(IMM8, 15),
2819                mask(IMM8, 16),
2820                mask(IMM8, 17),
2821                mask(IMM8, 18),
2822                mask(IMM8, 19),
2823                mask(IMM8, 20),
2824                mask(IMM8, 21),
2825                mask(IMM8, 22),
2826                mask(IMM8, 23),
2827                mask(IMM8, 24),
2828                mask(IMM8, 25),
2829                mask(IMM8, 26),
2830                mask(IMM8, 27),
2831                mask(IMM8, 28),
2832                mask(IMM8, 29),
2833                mask(IMM8, 30),
2834                mask(IMM8, 31),
2835            ],
2836        );
2837        transmute(r)
2838    }
2839}
2840
2841/// Shifts packed 32-bit integers in `a` left by the amount
2842/// specified by the corresponding element in `count` while
2843/// shifting in zeros, and returns the result.
2844///
2845/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_sllv_epi32)
2846#[inline]
2847#[target_feature(enable = "avx2")]
2848#[cfg_attr(test, assert_instr(vpsllvd))]
2849#[stable(feature = "simd_x86", since = "1.27.0")]
2850pub fn _mm_sllv_epi32(a: __m128i, count: __m128i) -> __m128i {
2851    unsafe { transmute(psllvd(a.as_i32x4(), count.as_i32x4())) }
2852}
2853
2854/// Shifts packed 32-bit integers in `a` left by the amount
2855/// specified by the corresponding element in `count` while
2856/// shifting in zeros, and returns the result.
2857///
2858/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm256_sllv_epi32)
2859#[inline]
2860#[target_feature(enable = "avx2")]
2861#[cfg_attr(test, assert_instr(vpsllvd))]
2862#[stable(feature = "simd_x86", since = "1.27.0")]
2863pub fn _mm256_sllv_epi32(a: __m256i, count: __m256i) -> __m256i {
2864    unsafe { transmute(psllvd256(a.as_i32x8(), count.as_i32x8())) }
2865}
2866
2867/// Shifts packed 64-bit integers in `a` left by the amount
2868/// specified by the corresponding element in `count` while
2869/// shifting in zeros, and returns the result.
2870///
2871/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_sllv_epi64)
2872#[inline]
2873#[target_feature(enable = "avx2")]
2874#[cfg_attr(test, assert_instr(vpsllvq))]
2875#[stable(feature = "simd_x86", since = "1.27.0")]
2876pub fn _mm_sllv_epi64(a: __m128i, count: __m128i) -> __m128i {
2877    unsafe { transmute(psllvq(a.as_i64x2(), count.as_i64x2())) }
2878}
2879
2880/// Shifts packed 64-bit integers in `a` left by the amount
2881/// specified by the corresponding element in `count` while
2882/// shifting in zeros, and returns the result.
2883///
2884/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm256_sllv_epi64)
2885#[inline]
2886#[target_feature(enable = "avx2")]
2887#[cfg_attr(test, assert_instr(vpsllvq))]
2888#[stable(feature = "simd_x86", since = "1.27.0")]
2889pub fn _mm256_sllv_epi64(a: __m256i, count: __m256i) -> __m256i {
2890    unsafe { transmute(psllvq256(a.as_i64x4(), count.as_i64x4())) }
2891}
2892
2893/// Shifts packed 16-bit integers in `a` right by `count` while
2894/// shifting in sign bits.
2895///
2896/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm256_sra_epi16)
2897#[inline]
2898#[target_feature(enable = "avx2")]
2899#[cfg_attr(test, assert_instr(vpsraw))]
2900#[stable(feature = "simd_x86", since = "1.27.0")]
2901pub fn _mm256_sra_epi16(a: __m256i, count: __m128i) -> __m256i {
2902    unsafe { transmute(psraw(a.as_i16x16(), count.as_i16x8())) }
2903}
2904
2905/// Shifts packed 32-bit integers in `a` right by `count` while
2906/// shifting in sign bits.
2907///
2908/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm256_sra_epi32)
2909#[inline]
2910#[target_feature(enable = "avx2")]
2911#[cfg_attr(test, assert_instr(vpsrad))]
2912#[stable(feature = "simd_x86", since = "1.27.0")]
2913pub fn _mm256_sra_epi32(a: __m256i, count: __m128i) -> __m256i {
2914    unsafe { transmute(psrad(a.as_i32x8(), count.as_i32x4())) }
2915}
2916
2917/// Shifts packed 16-bit integers in `a` right by `IMM8` while
2918/// shifting in sign bits.
2919///
2920/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm256_srai_epi16)
2921#[inline]
2922#[target_feature(enable = "avx2")]
2923#[cfg_attr(test, assert_instr(vpsraw, IMM8 = 7))]
2924#[rustc_legacy_const_generics(1)]
2925#[stable(feature = "simd_x86", since = "1.27.0")]
2926#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")]
2927pub const fn _mm256_srai_epi16<const IMM8: i32>(a: __m256i) -> __m256i {
2928    static_assert_uimm_bits!(IMM8, 8);
2929    unsafe { transmute(simd_shr(a.as_i16x16(), i16x16::splat(IMM8.min(15) as i16))) }
2930}
2931
2932/// Shifts packed 32-bit integers in `a` right by `IMM8` while
2933/// shifting in sign bits.
2934///
2935/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm256_srai_epi32)
2936#[inline]
2937#[target_feature(enable = "avx2")]
2938#[cfg_attr(test, assert_instr(vpsrad, IMM8 = 7))]
2939#[rustc_legacy_const_generics(1)]
2940#[stable(feature = "simd_x86", since = "1.27.0")]
2941#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")]
2942pub const fn _mm256_srai_epi32<const IMM8: i32>(a: __m256i) -> __m256i {
2943    static_assert_uimm_bits!(IMM8, 8);
2944    unsafe { transmute(simd_shr(a.as_i32x8(), i32x8::splat(IMM8.min(31)))) }
2945}
2946
2947/// Shifts packed 32-bit integers in `a` right by the amount specified by the
2948/// corresponding element in `count` while shifting in sign bits.
2949///
2950/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_srav_epi32)
2951#[inline]
2952#[target_feature(enable = "avx2")]
2953#[cfg_attr(test, assert_instr(vpsravd))]
2954#[stable(feature = "simd_x86", since = "1.27.0")]
2955pub fn _mm_srav_epi32(a: __m128i, count: __m128i) -> __m128i {
2956    unsafe { transmute(psravd(a.as_i32x4(), count.as_i32x4())) }
2957}
2958
2959/// Shifts packed 32-bit integers in `a` right by the amount specified by the
2960/// corresponding element in `count` while shifting in sign bits.
2961///
2962/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm256_srav_epi32)
2963#[inline]
2964#[target_feature(enable = "avx2")]
2965#[cfg_attr(test, assert_instr(vpsravd))]
2966#[stable(feature = "simd_x86", since = "1.27.0")]
2967pub fn _mm256_srav_epi32(a: __m256i, count: __m256i) -> __m256i {
2968    unsafe { transmute(psravd256(a.as_i32x8(), count.as_i32x8())) }
2969}
2970
2971/// Shifts 128-bit lanes in `a` right by `imm8` bytes while shifting in zeros.
2972///
2973/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm256_srli_si256)
2974#[inline]
2975#[target_feature(enable = "avx2")]
2976#[cfg_attr(test, assert_instr(vpsrldq, IMM8 = 1))]
2977#[rustc_legacy_const_generics(1)]
2978#[stable(feature = "simd_x86", since = "1.27.0")]
2979#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")]
2980pub const fn _mm256_srli_si256<const IMM8: i32>(a: __m256i) -> __m256i {
2981    static_assert_uimm_bits!(IMM8, 8);
2982    _mm256_bsrli_epi128::<IMM8>(a)
2983}
2984
2985/// Shifts 128-bit lanes in `a` right by `imm8` bytes while shifting in zeros.
2986///
2987/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm256_bsrli_epi128)
2988#[inline]
2989#[target_feature(enable = "avx2")]
2990#[cfg_attr(test, assert_instr(vpsrldq, IMM8 = 1))]
2991#[rustc_legacy_const_generics(1)]
2992#[stable(feature = "simd_x86", since = "1.27.0")]
2993#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")]
2994pub const fn _mm256_bsrli_epi128<const IMM8: i32>(a: __m256i) -> __m256i {
2995    static_assert_uimm_bits!(IMM8, 8);
2996    const fn mask(shift: i32, i: u32) -> u32 {
2997        let shift = shift as u32 & 0xff;
2998        if shift > 15 || (15 - (i % 16)) < shift {
2999            0
3000        } else {
3001            32 + (i + shift)
3002        }
3003    }
3004    unsafe {
3005        let a = a.as_i8x32();
3006        let r: i8x32 = simd_shuffle!(
3007            i8x32::ZERO,
3008            a,
3009            [
3010                mask(IMM8, 0),
3011                mask(IMM8, 1),
3012                mask(IMM8, 2),
3013                mask(IMM8, 3),
3014                mask(IMM8, 4),
3015                mask(IMM8, 5),
3016                mask(IMM8, 6),
3017                mask(IMM8, 7),
3018                mask(IMM8, 8),
3019                mask(IMM8, 9),
3020                mask(IMM8, 10),
3021                mask(IMM8, 11),
3022                mask(IMM8, 12),
3023                mask(IMM8, 13),
3024                mask(IMM8, 14),
3025                mask(IMM8, 15),
3026                mask(IMM8, 16),
3027                mask(IMM8, 17),
3028                mask(IMM8, 18),
3029                mask(IMM8, 19),
3030                mask(IMM8, 20),
3031                mask(IMM8, 21),
3032                mask(IMM8, 22),
3033                mask(IMM8, 23),
3034                mask(IMM8, 24),
3035                mask(IMM8, 25),
3036                mask(IMM8, 26),
3037                mask(IMM8, 27),
3038                mask(IMM8, 28),
3039                mask(IMM8, 29),
3040                mask(IMM8, 30),
3041                mask(IMM8, 31),
3042            ],
3043        );
3044        transmute(r)
3045    }
3046}
3047
3048/// Shifts packed 16-bit integers in `a` right by `count` while shifting in
3049/// zeros.
3050///
3051/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm256_srl_epi16)
3052#[inline]
3053#[target_feature(enable = "avx2")]
3054#[cfg_attr(test, assert_instr(vpsrlw))]
3055#[stable(feature = "simd_x86", since = "1.27.0")]
3056pub fn _mm256_srl_epi16(a: __m256i, count: __m128i) -> __m256i {
3057    unsafe { transmute(psrlw(a.as_i16x16(), count.as_i16x8())) }
3058}
3059
3060/// Shifts packed 32-bit integers in `a` right by `count` while shifting in
3061/// zeros.
3062///
3063/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm256_srl_epi32)
3064#[inline]
3065#[target_feature(enable = "avx2")]
3066#[cfg_attr(test, assert_instr(vpsrld))]
3067#[stable(feature = "simd_x86", since = "1.27.0")]
3068pub fn _mm256_srl_epi32(a: __m256i, count: __m128i) -> __m256i {
3069    unsafe { transmute(psrld(a.as_i32x8(), count.as_i32x4())) }
3070}
3071
3072/// Shifts packed 64-bit integers in `a` right by `count` while shifting in
3073/// zeros.
3074///
3075/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm256_srl_epi64)
3076#[inline]
3077#[target_feature(enable = "avx2")]
3078#[cfg_attr(test, assert_instr(vpsrlq))]
3079#[stable(feature = "simd_x86", since = "1.27.0")]
3080pub fn _mm256_srl_epi64(a: __m256i, count: __m128i) -> __m256i {
3081    unsafe { transmute(psrlq(a.as_i64x4(), count.as_i64x2())) }
3082}
3083
3084/// Shifts packed 16-bit integers in `a` right by `IMM8` while shifting in
3085/// zeros
3086///
3087/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm256_srli_epi16)
3088#[inline]
3089#[target_feature(enable = "avx2")]
3090#[cfg_attr(test, assert_instr(vpsrlw, IMM8 = 7))]
3091#[rustc_legacy_const_generics(1)]
3092#[stable(feature = "simd_x86", since = "1.27.0")]
3093#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")]
3094pub const fn _mm256_srli_epi16<const IMM8: i32>(a: __m256i) -> __m256i {
3095    static_assert_uimm_bits!(IMM8, 8);
3096    unsafe {
3097        if IMM8 >= 16 {
3098            _mm256_setzero_si256()
3099        } else {
3100            transmute(simd_shr(a.as_u16x16(), u16x16::splat(IMM8 as u16)))
3101        }
3102    }
3103}
3104
3105/// Shifts packed 32-bit integers in `a` right by `IMM8` while shifting in
3106/// zeros
3107///
3108/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm256_srli_epi32)
3109#[inline]
3110#[target_feature(enable = "avx2")]
3111#[cfg_attr(test, assert_instr(vpsrld, IMM8 = 7))]
3112#[rustc_legacy_const_generics(1)]
3113#[stable(feature = "simd_x86", since = "1.27.0")]
3114#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")]
3115pub const fn _mm256_srli_epi32<const IMM8: i32>(a: __m256i) -> __m256i {
3116    static_assert_uimm_bits!(IMM8, 8);
3117    unsafe {
3118        if IMM8 >= 32 {
3119            _mm256_setzero_si256()
3120        } else {
3121            transmute(simd_shr(a.as_u32x8(), u32x8::splat(IMM8 as u32)))
3122        }
3123    }
3124}
3125
3126/// Shifts packed 64-bit integers in `a` right by `IMM8` while shifting in
3127/// zeros
3128///
3129/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm256_srli_epi64)
3130#[inline]
3131#[target_feature(enable = "avx2")]
3132#[cfg_attr(test, assert_instr(vpsrlq, IMM8 = 7))]
3133#[rustc_legacy_const_generics(1)]
3134#[stable(feature = "simd_x86", since = "1.27.0")]
3135#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")]
3136pub const fn _mm256_srli_epi64<const IMM8: i32>(a: __m256i) -> __m256i {
3137    static_assert_uimm_bits!(IMM8, 8);
3138    unsafe {
3139        if IMM8 >= 64 {
3140            _mm256_setzero_si256()
3141        } else {
3142            transmute(simd_shr(a.as_u64x4(), u64x4::splat(IMM8 as u64)))
3143        }
3144    }
3145}
3146
3147/// Shifts packed 32-bit integers in `a` right by the amount specified by
3148/// the corresponding element in `count` while shifting in zeros,
3149///
3150/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_srlv_epi32)
3151#[inline]
3152#[target_feature(enable = "avx2")]
3153#[cfg_attr(test, assert_instr(vpsrlvd))]
3154#[stable(feature = "simd_x86", since = "1.27.0")]
3155pub fn _mm_srlv_epi32(a: __m128i, count: __m128i) -> __m128i {
3156    unsafe { transmute(psrlvd(a.as_i32x4(), count.as_i32x4())) }
3157}
3158
3159/// Shifts packed 32-bit integers in `a` right by the amount specified by
3160/// the corresponding element in `count` while shifting in zeros,
3161///
3162/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm256_srlv_epi32)
3163#[inline]
3164#[target_feature(enable = "avx2")]
3165#[cfg_attr(test, assert_instr(vpsrlvd))]
3166#[stable(feature = "simd_x86", since = "1.27.0")]
3167pub fn _mm256_srlv_epi32(a: __m256i, count: __m256i) -> __m256i {
3168    unsafe { transmute(psrlvd256(a.as_i32x8(), count.as_i32x8())) }
3169}
3170
3171/// Shifts packed 64-bit integers in `a` right by the amount specified by
3172/// the corresponding element in `count` while shifting in zeros,
3173///
3174/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_srlv_epi64)
3175#[inline]
3176#[target_feature(enable = "avx2")]
3177#[cfg_attr(test, assert_instr(vpsrlvq))]
3178#[stable(feature = "simd_x86", since = "1.27.0")]
3179pub fn _mm_srlv_epi64(a: __m128i, count: __m128i) -> __m128i {
3180    unsafe { transmute(psrlvq(a.as_i64x2(), count.as_i64x2())) }
3181}
3182
3183/// Shifts packed 64-bit integers in `a` right by the amount specified by
3184/// the corresponding element in `count` while shifting in zeros,
3185///
3186/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm256_srlv_epi64)
3187#[inline]
3188#[target_feature(enable = "avx2")]
3189#[cfg_attr(test, assert_instr(vpsrlvq))]
3190#[stable(feature = "simd_x86", since = "1.27.0")]
3191pub fn _mm256_srlv_epi64(a: __m256i, count: __m256i) -> __m256i {
3192    unsafe { transmute(psrlvq256(a.as_i64x4(), count.as_i64x4())) }
3193}
3194
3195/// Load 256-bits of integer data from memory into dst using a non-temporal memory hint. mem_addr
3196/// must be aligned on a 32-byte boundary or a general-protection exception may be generated. To
3197/// minimize caching, the data is flagged as non-temporal (unlikely to be used again soon)
3198///
3199/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm256_stream_load_si256)
3200#[inline]
3201#[target_feature(enable = "avx2")]
3202#[cfg_attr(test, assert_instr(vmovntdqa))]
3203#[stable(feature = "simd_x86_updates", since = "1.82.0")]
3204pub unsafe fn _mm256_stream_load_si256(mem_addr: *const __m256i) -> __m256i {
3205    let dst: __m256i;
3206    crate::arch::asm!(
3207        vpl!("vmovntdqa {a}"),
3208        a = out(ymm_reg) dst,
3209        p = in(reg) mem_addr,
3210        options(pure, readonly, nostack, preserves_flags),
3211    );
3212    dst
3213}
3214
3215/// Subtract packed 16-bit integers in `b` from packed 16-bit integers in `a`
3216///
3217/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm256_sub_epi16)
3218#[inline]
3219#[target_feature(enable = "avx2")]
3220#[cfg_attr(test, assert_instr(vpsubw))]
3221#[stable(feature = "simd_x86", since = "1.27.0")]
3222#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")]
3223pub const fn _mm256_sub_epi16(a: __m256i, b: __m256i) -> __m256i {
3224    unsafe { transmute(simd_sub(a.as_i16x16(), b.as_i16x16())) }
3225}
3226
3227/// Subtract packed 32-bit integers in `b` from packed 32-bit integers in `a`
3228///
3229/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm256_sub_epi32)
3230#[inline]
3231#[target_feature(enable = "avx2")]
3232#[cfg_attr(test, assert_instr(vpsubd))]
3233#[stable(feature = "simd_x86", since = "1.27.0")]
3234#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")]
3235pub const fn _mm256_sub_epi32(a: __m256i, b: __m256i) -> __m256i {
3236    unsafe { transmute(simd_sub(a.as_i32x8(), b.as_i32x8())) }
3237}
3238
3239/// Subtract packed 64-bit integers in `b` from packed 64-bit integers in `a`
3240///
3241/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm256_sub_epi64)
3242#[inline]
3243#[target_feature(enable = "avx2")]
3244#[cfg_attr(test, assert_instr(vpsubq))]
3245#[stable(feature = "simd_x86", since = "1.27.0")]
3246#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")]
3247pub const fn _mm256_sub_epi64(a: __m256i, b: __m256i) -> __m256i {
3248    unsafe { transmute(simd_sub(a.as_i64x4(), b.as_i64x4())) }
3249}
3250
3251/// Subtract packed 8-bit integers in `b` from packed 8-bit integers in `a`
3252///
3253/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm256_sub_epi8)
3254#[inline]
3255#[target_feature(enable = "avx2")]
3256#[cfg_attr(test, assert_instr(vpsubb))]
3257#[stable(feature = "simd_x86", since = "1.27.0")]
3258#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")]
3259pub const fn _mm256_sub_epi8(a: __m256i, b: __m256i) -> __m256i {
3260    unsafe { transmute(simd_sub(a.as_i8x32(), b.as_i8x32())) }
3261}
3262
3263/// Subtract packed 16-bit integers in `b` from packed 16-bit integers in
3264/// `a` using saturation.
3265///
3266/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm256_subs_epi16)
3267#[inline]
3268#[target_feature(enable = "avx2")]
3269#[cfg_attr(test, assert_instr(vpsubsw))]
3270#[stable(feature = "simd_x86", since = "1.27.0")]
3271#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")]
3272pub const fn _mm256_subs_epi16(a: __m256i, b: __m256i) -> __m256i {
3273    unsafe { transmute(simd_saturating_sub(a.as_i16x16(), b.as_i16x16())) }
3274}
3275
3276/// Subtract packed 8-bit integers in `b` from packed 8-bit integers in
3277/// `a` using saturation.
3278///
3279/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm256_subs_epi8)
3280#[inline]
3281#[target_feature(enable = "avx2")]
3282#[cfg_attr(test, assert_instr(vpsubsb))]
3283#[stable(feature = "simd_x86", since = "1.27.0")]
3284#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")]
3285pub const fn _mm256_subs_epi8(a: __m256i, b: __m256i) -> __m256i {
3286    unsafe { transmute(simd_saturating_sub(a.as_i8x32(), b.as_i8x32())) }
3287}
3288
3289/// Subtract packed unsigned 16-bit integers in `b` from packed 16-bit
3290/// integers in `a` using saturation.
3291///
3292/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm256_subs_epu16)
3293#[inline]
3294#[target_feature(enable = "avx2")]
3295#[cfg_attr(test, assert_instr(vpsubusw))]
3296#[stable(feature = "simd_x86", since = "1.27.0")]
3297#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")]
3298pub const fn _mm256_subs_epu16(a: __m256i, b: __m256i) -> __m256i {
3299    unsafe { transmute(simd_saturating_sub(a.as_u16x16(), b.as_u16x16())) }
3300}
3301
3302/// Subtract packed unsigned 8-bit integers in `b` from packed 8-bit
3303/// integers in `a` using saturation.
3304///
3305/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm256_subs_epu8)
3306#[inline]
3307#[target_feature(enable = "avx2")]
3308#[cfg_attr(test, assert_instr(vpsubusb))]
3309#[stable(feature = "simd_x86", since = "1.27.0")]
3310#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")]
3311pub const fn _mm256_subs_epu8(a: __m256i, b: __m256i) -> __m256i {
3312    unsafe { transmute(simd_saturating_sub(a.as_u8x32(), b.as_u8x32())) }
3313}
3314
3315/// Unpacks and interleave 8-bit integers from the high half of each
3316/// 128-bit lane in `a` and `b`.
3317///
3318/// ```rust
3319/// #[cfg(target_arch = "x86")]
3320/// use std::arch::x86::*;
3321/// #[cfg(target_arch = "x86_64")]
3322/// use std::arch::x86_64::*;
3323///
3324/// # fn main() {
3325/// #     if is_x86_feature_detected!("avx2") {
3326/// #         #[target_feature(enable = "avx2")]
3327/// #         unsafe fn worker() {
3328/// let a = _mm256_setr_epi8(
3329///     0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19,
3330///     20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31,
3331/// );
3332/// let b = _mm256_setr_epi8(
3333///     0, -1, -2, -3, -4, -5, -6, -7, -8, -9, -10, -11, -12, -13, -14, -15,
3334///     -16, -17, -18, -19, -20, -21, -22, -23, -24, -25, -26, -27, -28, -29,
3335///     -30, -31,
3336/// );
3337///
3338/// let c = _mm256_unpackhi_epi8(a, b);
3339///
3340/// let expected = _mm256_setr_epi8(
3341///     8, -8, 9, -9, 10, -10, 11, -11, 12, -12, 13, -13, 14, -14, 15, -15,
3342///     24, -24, 25, -25, 26, -26, 27, -27, 28, -28, 29, -29, 30, -30, 31,
3343///     -31,
3344/// );
3345/// assert_eq!(_mm256_movemask_epi8(_mm256_cmpeq_epi8(c, expected)), !0);
3346///
3347/// #         }
3348/// #         unsafe { worker(); }
3349/// #     }
3350/// # }
3351/// ```
3352///
3353/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm256_unpackhi_epi8)
3354#[inline]
3355#[target_feature(enable = "avx2")]
3356#[cfg_attr(test, assert_instr(vpunpckhbw))]
3357#[stable(feature = "simd_x86", since = "1.27.0")]
3358#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")]
3359pub const fn _mm256_unpackhi_epi8(a: __m256i, b: __m256i) -> __m256i {
3360    unsafe {
3361        #[rustfmt::skip]
3362        let r: i8x32 = simd_shuffle!(a.as_i8x32(), b.as_i8x32(), [
3363                8, 40, 9, 41, 10, 42, 11, 43,
3364                12, 44, 13, 45, 14, 46, 15, 47,
3365                24, 56, 25, 57, 26, 58, 27, 59,
3366                28, 60, 29, 61, 30, 62, 31, 63,
3367        ]);
3368        transmute(r)
3369    }
3370}
3371
3372/// Unpacks and interleave 8-bit integers from the low half of each
3373/// 128-bit lane of `a` and `b`.
3374///
3375/// ```rust
3376/// #[cfg(target_arch = "x86")]
3377/// use std::arch::x86::*;
3378/// #[cfg(target_arch = "x86_64")]
3379/// use std::arch::x86_64::*;
3380///
3381/// # fn main() {
3382/// #     if is_x86_feature_detected!("avx2") {
3383/// #         #[target_feature(enable = "avx2")]
3384/// #         unsafe fn worker() {
3385/// let a = _mm256_setr_epi8(
3386///     0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19,
3387///     20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31,
3388/// );
3389/// let b = _mm256_setr_epi8(
3390///     0, -1, -2, -3, -4, -5, -6, -7, -8, -9, -10, -11, -12, -13, -14, -15,
3391///     -16, -17, -18, -19, -20, -21, -22, -23, -24, -25, -26, -27, -28, -29,
3392///     -30, -31,
3393/// );
3394///
3395/// let c = _mm256_unpacklo_epi8(a, b);
3396///
3397/// let expected = _mm256_setr_epi8(
3398///     0, 0, 1, -1, 2, -2, 3, -3, 4, -4, 5, -5, 6, -6, 7, -7, 16, -16, 17,
3399///     -17, 18, -18, 19, -19, 20, -20, 21, -21, 22, -22, 23, -23,
3400/// );
3401/// assert_eq!(_mm256_movemask_epi8(_mm256_cmpeq_epi8(c, expected)), !0);
3402///
3403/// #         }
3404/// #         unsafe { worker(); }
3405/// #     }
3406/// # }
3407/// ```
3408///
3409/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm256_unpacklo_epi8)
3410#[inline]
3411#[target_feature(enable = "avx2")]
3412#[cfg_attr(test, assert_instr(vpunpcklbw))]
3413#[stable(feature = "simd_x86", since = "1.27.0")]
3414#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")]
3415pub const fn _mm256_unpacklo_epi8(a: __m256i, b: __m256i) -> __m256i {
3416    unsafe {
3417        #[rustfmt::skip]
3418        let r: i8x32 = simd_shuffle!(a.as_i8x32(), b.as_i8x32(), [
3419            0, 32, 1, 33, 2, 34, 3, 35,
3420            4, 36, 5, 37, 6, 38, 7, 39,
3421            16, 48, 17, 49, 18, 50, 19, 51,
3422            20, 52, 21, 53, 22, 54, 23, 55,
3423        ]);
3424        transmute(r)
3425    }
3426}
3427
3428/// Unpacks and interleave 16-bit integers from the high half of each
3429/// 128-bit lane of `a` and `b`.
3430///
3431/// ```rust
3432/// #[cfg(target_arch = "x86")]
3433/// use std::arch::x86::*;
3434/// #[cfg(target_arch = "x86_64")]
3435/// use std::arch::x86_64::*;
3436///
3437/// # fn main() {
3438/// #     if is_x86_feature_detected!("avx2") {
3439/// #         #[target_feature(enable = "avx2")]
3440/// #         unsafe fn worker() {
3441/// let a = _mm256_setr_epi16(
3442///     0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15,
3443/// );
3444/// let b = _mm256_setr_epi16(
3445///     0, -1, -2, -3, -4, -5, -6, -7, -8, -9, -10, -11, -12, -13, -14, -15,
3446/// );
3447///
3448/// let c = _mm256_unpackhi_epi16(a, b);
3449///
3450/// let expected = _mm256_setr_epi16(
3451///     4, -4, 5, -5, 6, -6, 7, -7, 12, -12, 13, -13, 14, -14, 15, -15,
3452/// );
3453/// assert_eq!(_mm256_movemask_epi8(_mm256_cmpeq_epi8(c, expected)), !0);
3454///
3455/// #         }
3456/// #         unsafe { worker(); }
3457/// #     }
3458/// # }
3459/// ```
3460///
3461/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm256_unpackhi_epi16)
3462#[inline]
3463#[target_feature(enable = "avx2")]
3464#[cfg_attr(test, assert_instr(vpunpckhwd))]
3465#[stable(feature = "simd_x86", since = "1.27.0")]
3466#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")]
3467pub const fn _mm256_unpackhi_epi16(a: __m256i, b: __m256i) -> __m256i {
3468    unsafe {
3469        let r: i16x16 = simd_shuffle!(
3470            a.as_i16x16(),
3471            b.as_i16x16(),
3472            [4, 20, 5, 21, 6, 22, 7, 23, 12, 28, 13, 29, 14, 30, 15, 31],
3473        );
3474        transmute(r)
3475    }
3476}
3477
3478/// Unpacks and interleave 16-bit integers from the low half of each
3479/// 128-bit lane of `a` and `b`.
3480///
3481/// ```rust
3482/// #[cfg(target_arch = "x86")]
3483/// use std::arch::x86::*;
3484/// #[cfg(target_arch = "x86_64")]
3485/// use std::arch::x86_64::*;
3486///
3487/// # fn main() {
3488/// #     if is_x86_feature_detected!("avx2") {
3489/// #         #[target_feature(enable = "avx2")]
3490/// #         unsafe fn worker() {
3491///
3492/// let a = _mm256_setr_epi16(
3493///     0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15,
3494/// );
3495/// let b = _mm256_setr_epi16(
3496///     0, -1, -2, -3, -4, -5, -6, -7, -8, -9, -10, -11, -12, -13, -14, -15,
3497/// );
3498///
3499/// let c = _mm256_unpacklo_epi16(a, b);
3500///
3501/// let expected = _mm256_setr_epi16(
3502///     0, 0, 1, -1, 2, -2, 3, -3, 8, -8, 9, -9, 10, -10, 11, -11,
3503/// );
3504/// assert_eq!(_mm256_movemask_epi8(_mm256_cmpeq_epi8(c, expected)), !0);
3505///
3506/// #         }
3507/// #         unsafe { worker(); }
3508/// #     }
3509/// # }
3510/// ```
3511///
3512/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm256_unpacklo_epi16)
3513#[inline]
3514#[target_feature(enable = "avx2")]
3515#[cfg_attr(test, assert_instr(vpunpcklwd))]
3516#[stable(feature = "simd_x86", since = "1.27.0")]
3517#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")]
3518pub const fn _mm256_unpacklo_epi16(a: __m256i, b: __m256i) -> __m256i {
3519    unsafe {
3520        let r: i16x16 = simd_shuffle!(
3521            a.as_i16x16(),
3522            b.as_i16x16(),
3523            [0, 16, 1, 17, 2, 18, 3, 19, 8, 24, 9, 25, 10, 26, 11, 27],
3524        );
3525        transmute(r)
3526    }
3527}
3528
3529/// Unpacks and interleave 32-bit integers from the high half of each
3530/// 128-bit lane of `a` and `b`.
3531///
3532/// ```rust
3533/// #[cfg(target_arch = "x86")]
3534/// use std::arch::x86::*;
3535/// #[cfg(target_arch = "x86_64")]
3536/// use std::arch::x86_64::*;
3537///
3538/// # fn main() {
3539/// #     if is_x86_feature_detected!("avx2") {
3540/// #         #[target_feature(enable = "avx2")]
3541/// #         unsafe fn worker() {
3542/// let a = _mm256_setr_epi32(0, 1, 2, 3, 4, 5, 6, 7);
3543/// let b = _mm256_setr_epi32(0, -1, -2, -3, -4, -5, -6, -7);
3544///
3545/// let c = _mm256_unpackhi_epi32(a, b);
3546///
3547/// let expected = _mm256_setr_epi32(2, -2, 3, -3, 6, -6, 7, -7);
3548/// assert_eq!(_mm256_movemask_epi8(_mm256_cmpeq_epi8(c, expected)), !0);
3549///
3550/// #         }
3551/// #         unsafe { worker(); }
3552/// #     }
3553/// # }
3554/// ```
3555///
3556/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm256_unpackhi_epi32)
3557#[inline]
3558#[target_feature(enable = "avx2")]
3559#[cfg_attr(test, assert_instr(vunpckhps))]
3560#[stable(feature = "simd_x86", since = "1.27.0")]
3561#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")]
3562pub const fn _mm256_unpackhi_epi32(a: __m256i, b: __m256i) -> __m256i {
3563    unsafe {
3564        let r: i32x8 = simd_shuffle!(a.as_i32x8(), b.as_i32x8(), [2, 10, 3, 11, 6, 14, 7, 15]);
3565        transmute(r)
3566    }
3567}
3568
3569/// Unpacks and interleave 32-bit integers from the low half of each
3570/// 128-bit lane of `a` and `b`.
3571///
3572/// ```rust
3573/// #[cfg(target_arch = "x86")]
3574/// use std::arch::x86::*;
3575/// #[cfg(target_arch = "x86_64")]
3576/// use std::arch::x86_64::*;
3577///
3578/// # fn main() {
3579/// #     if is_x86_feature_detected!("avx2") {
3580/// #         #[target_feature(enable = "avx2")]
3581/// #         unsafe fn worker() {
3582/// let a = _mm256_setr_epi32(0, 1, 2, 3, 4, 5, 6, 7);
3583/// let b = _mm256_setr_epi32(0, -1, -2, -3, -4, -5, -6, -7);
3584///
3585/// let c = _mm256_unpacklo_epi32(a, b);
3586///
3587/// let expected = _mm256_setr_epi32(0, 0, 1, -1, 4, -4, 5, -5);
3588/// assert_eq!(_mm256_movemask_epi8(_mm256_cmpeq_epi8(c, expected)), !0);
3589///
3590/// #         }
3591/// #         unsafe { worker(); }
3592/// #     }
3593/// # }
3594/// ```
3595///
3596/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm256_unpacklo_epi32)
3597#[inline]
3598#[target_feature(enable = "avx2")]
3599#[cfg_attr(test, assert_instr(vunpcklps))]
3600#[stable(feature = "simd_x86", since = "1.27.0")]
3601#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")]
3602pub const fn _mm256_unpacklo_epi32(a: __m256i, b: __m256i) -> __m256i {
3603    unsafe {
3604        let r: i32x8 = simd_shuffle!(a.as_i32x8(), b.as_i32x8(), [0, 8, 1, 9, 4, 12, 5, 13]);
3605        transmute(r)
3606    }
3607}
3608
3609/// Unpacks and interleave 64-bit integers from the high half of each
3610/// 128-bit lane of `a` and `b`.
3611///
3612/// ```rust
3613/// #[cfg(target_arch = "x86")]
3614/// use std::arch::x86::*;
3615/// #[cfg(target_arch = "x86_64")]
3616/// use std::arch::x86_64::*;
3617///
3618/// # fn main() {
3619/// #     if is_x86_feature_detected!("avx2") {
3620/// #         #[target_feature(enable = "avx2")]
3621/// #         unsafe fn worker() {
3622/// let a = _mm256_setr_epi64x(0, 1, 2, 3);
3623/// let b = _mm256_setr_epi64x(0, -1, -2, -3);
3624///
3625/// let c = _mm256_unpackhi_epi64(a, b);
3626///
3627/// let expected = _mm256_setr_epi64x(1, -1, 3, -3);
3628/// assert_eq!(_mm256_movemask_epi8(_mm256_cmpeq_epi8(c, expected)), !0);
3629///
3630/// #         }
3631/// #         unsafe { worker(); }
3632/// #     }
3633/// # }
3634/// ```
3635///
3636/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm256_unpackhi_epi64)
3637#[inline]
3638#[target_feature(enable = "avx2")]
3639#[cfg_attr(test, assert_instr(vunpckhpd))]
3640#[stable(feature = "simd_x86", since = "1.27.0")]
3641#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")]
3642pub const fn _mm256_unpackhi_epi64(a: __m256i, b: __m256i) -> __m256i {
3643    unsafe {
3644        let r: i64x4 = simd_shuffle!(a.as_i64x4(), b.as_i64x4(), [1, 5, 3, 7]);
3645        transmute(r)
3646    }
3647}
3648
3649/// Unpacks and interleave 64-bit integers from the low half of each
3650/// 128-bit lane of `a` and `b`.
3651///
3652/// ```rust
3653/// #[cfg(target_arch = "x86")]
3654/// use std::arch::x86::*;
3655/// #[cfg(target_arch = "x86_64")]
3656/// use std::arch::x86_64::*;
3657///
3658/// # fn main() {
3659/// #     if is_x86_feature_detected!("avx2") {
3660/// #         #[target_feature(enable = "avx2")]
3661/// #         unsafe fn worker() {
3662/// let a = _mm256_setr_epi64x(0, 1, 2, 3);
3663/// let b = _mm256_setr_epi64x(0, -1, -2, -3);
3664///
3665/// let c = _mm256_unpacklo_epi64(a, b);
3666///
3667/// let expected = _mm256_setr_epi64x(0, 0, 2, -2);
3668/// assert_eq!(_mm256_movemask_epi8(_mm256_cmpeq_epi8(c, expected)), !0);
3669///
3670/// #         }
3671/// #         unsafe { worker(); }
3672/// #     }
3673/// # }
3674/// ```
3675///
3676/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm256_unpacklo_epi64)
3677#[inline]
3678#[target_feature(enable = "avx2")]
3679#[cfg_attr(test, assert_instr(vunpcklpd))]
3680#[stable(feature = "simd_x86", since = "1.27.0")]
3681#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")]
3682pub const fn _mm256_unpacklo_epi64(a: __m256i, b: __m256i) -> __m256i {
3683    unsafe {
3684        let r: i64x4 = simd_shuffle!(a.as_i64x4(), b.as_i64x4(), [0, 4, 2, 6]);
3685        transmute(r)
3686    }
3687}
3688
3689/// Computes the bitwise XOR of 256 bits (representing integer data)
3690/// in `a` and `b`
3691///
3692/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm256_xor_si256)
3693#[inline]
3694#[target_feature(enable = "avx2")]
3695#[cfg_attr(test, assert_instr(vxorps))]
3696#[stable(feature = "simd_x86", since = "1.27.0")]
3697#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")]
3698pub const fn _mm256_xor_si256(a: __m256i, b: __m256i) -> __m256i {
3699    unsafe { transmute(simd_xor(a.as_i64x4(), b.as_i64x4())) }
3700}
3701
3702/// Extracts an 8-bit integer from `a`, selected with `INDEX`. Returns a 32-bit
3703/// integer containing the zero-extended integer data.
3704///
3705/// See [LLVM commit D20468](https://reviews.llvm.org/D20468).
3706///
3707/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm256_extract_epi8)
3708#[inline]
3709#[target_feature(enable = "avx2")]
3710// This intrinsic has no corresponding instruction.
3711#[rustc_legacy_const_generics(1)]
3712#[stable(feature = "simd_x86", since = "1.27.0")]
3713#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")]
3714pub const fn _mm256_extract_epi8<const INDEX: i32>(a: __m256i) -> i32 {
3715    static_assert_uimm_bits!(INDEX, 5);
3716    unsafe { simd_extract!(a.as_u8x32(), INDEX as u32, u8) as i32 }
3717}
3718
3719/// Extracts a 16-bit integer from `a`, selected with `INDEX`. Returns a 32-bit
3720/// integer containing the zero-extended integer data.
3721///
3722/// See [LLVM commit D20468](https://reviews.llvm.org/D20468).
3723///
3724/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm256_extract_epi16)
3725#[inline]
3726#[target_feature(enable = "avx2")]
3727// This intrinsic has no corresponding instruction.
3728#[rustc_legacy_const_generics(1)]
3729#[stable(feature = "simd_x86", since = "1.27.0")]
3730#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")]
3731pub const fn _mm256_extract_epi16<const INDEX: i32>(a: __m256i) -> i32 {
3732    static_assert_uimm_bits!(INDEX, 4);
3733    unsafe { simd_extract!(a.as_u16x16(), INDEX as u32, u16) as i32 }
3734}
3735
3736#[allow(improper_ctypes)]
3737unsafe extern "llvm-intrinsic" {
3738    #[link_name = "llvm.x86.avx2.pavg.b"]
3739    fn pavgb(a: u8x32, b: u8x32) -> u8x32;
3740    #[link_name = "llvm.x86.avx2.pavg.w"]
3741    fn pavgw(a: u16x16, b: u16x16) -> u16x16;
3742    #[link_name = "llvm.x86.avx2.pmadd.wd"]
3743    fn pmaddwd(a: i16x16, b: i16x16) -> i32x8;
3744    #[link_name = "llvm.x86.avx2.pmadd.ub.sw"]
3745    fn pmaddubsw(a: u8x32, b: i8x32) -> i16x16;
3746    #[link_name = "llvm.x86.avx2.mpsadbw"]
3747    fn mpsadbw(a: u8x32, b: u8x32, imm8: i8) -> u16x16;
3748    #[link_name = "llvm.x86.avx2.pmulhu.w"]
3749    fn pmulhuw(a: u16x16, b: u16x16) -> u16x16;
3750    #[link_name = "llvm.x86.avx2.pmulh.w"]
3751    fn pmulhw(a: i16x16, b: i16x16) -> i16x16;
3752    #[link_name = "llvm.x86.avx2.pmul.hr.sw"]
3753    fn pmulhrsw(a: i16x16, b: i16x16) -> i16x16;
3754    #[link_name = "llvm.x86.avx2.packsswb"]
3755    fn packsswb(a: i16x16, b: i16x16) -> i8x32;
3756    #[link_name = "llvm.x86.avx2.packssdw"]
3757    fn packssdw(a: i32x8, b: i32x8) -> i16x16;
3758    #[link_name = "llvm.x86.avx2.packuswb"]
3759    fn packuswb(a: i16x16, b: i16x16) -> u8x32;
3760    #[link_name = "llvm.x86.avx2.packusdw"]
3761    fn packusdw(a: i32x8, b: i32x8) -> u16x16;
3762    #[link_name = "llvm.x86.avx2.psad.bw"]
3763    fn psadbw(a: u8x32, b: u8x32) -> u64x4;
3764    #[link_name = "llvm.x86.avx2.psign.b"]
3765    fn psignb(a: i8x32, b: i8x32) -> i8x32;
3766    #[link_name = "llvm.x86.avx2.psign.w"]
3767    fn psignw(a: i16x16, b: i16x16) -> i16x16;
3768    #[link_name = "llvm.x86.avx2.psign.d"]
3769    fn psignd(a: i32x8, b: i32x8) -> i32x8;
3770    #[link_name = "llvm.x86.avx2.psll.w"]
3771    fn psllw(a: i16x16, count: i16x8) -> i16x16;
3772    #[link_name = "llvm.x86.avx2.psll.d"]
3773    fn pslld(a: i32x8, count: i32x4) -> i32x8;
3774    #[link_name = "llvm.x86.avx2.psll.q"]
3775    fn psllq(a: i64x4, count: i64x2) -> i64x4;
3776    #[link_name = "llvm.x86.avx2.psllv.d"]
3777    fn psllvd(a: i32x4, count: i32x4) -> i32x4;
3778    #[link_name = "llvm.x86.avx2.psllv.d.256"]
3779    fn psllvd256(a: i32x8, count: i32x8) -> i32x8;
3780    #[link_name = "llvm.x86.avx2.psllv.q"]
3781    fn psllvq(a: i64x2, count: i64x2) -> i64x2;
3782    #[link_name = "llvm.x86.avx2.psllv.q.256"]
3783    fn psllvq256(a: i64x4, count: i64x4) -> i64x4;
3784    #[link_name = "llvm.x86.avx2.psra.w"]
3785    fn psraw(a: i16x16, count: i16x8) -> i16x16;
3786    #[link_name = "llvm.x86.avx2.psra.d"]
3787    fn psrad(a: i32x8, count: i32x4) -> i32x8;
3788    #[link_name = "llvm.x86.avx2.psrav.d"]
3789    fn psravd(a: i32x4, count: i32x4) -> i32x4;
3790    #[link_name = "llvm.x86.avx2.psrav.d.256"]
3791    fn psravd256(a: i32x8, count: i32x8) -> i32x8;
3792    #[link_name = "llvm.x86.avx2.psrl.w"]
3793    fn psrlw(a: i16x16, count: i16x8) -> i16x16;
3794    #[link_name = "llvm.x86.avx2.psrl.d"]
3795    fn psrld(a: i32x8, count: i32x4) -> i32x8;
3796    #[link_name = "llvm.x86.avx2.psrl.q"]
3797    fn psrlq(a: i64x4, count: i64x2) -> i64x4;
3798    #[link_name = "llvm.x86.avx2.psrlv.d"]
3799    fn psrlvd(a: i32x4, count: i32x4) -> i32x4;
3800    #[link_name = "llvm.x86.avx2.psrlv.d.256"]
3801    fn psrlvd256(a: i32x8, count: i32x8) -> i32x8;
3802    #[link_name = "llvm.x86.avx2.psrlv.q"]
3803    fn psrlvq(a: i64x2, count: i64x2) -> i64x2;
3804    #[link_name = "llvm.x86.avx2.psrlv.q.256"]
3805    fn psrlvq256(a: i64x4, count: i64x4) -> i64x4;
3806    #[link_name = "llvm.x86.avx2.pshuf.b"]
3807    fn pshufb(a: u8x32, b: u8x32) -> u8x32;
3808    #[link_name = "llvm.x86.avx2.permd"]
3809    fn permd(a: u32x8, b: u32x8) -> u32x8;
3810    #[link_name = "llvm.x86.avx2.permps"]
3811    fn permps(a: __m256, b: i32x8) -> __m256;
3812    #[link_name = "llvm.x86.avx2.gather.d.d"]
3813    fn pgatherdd(src: i32x4, slice: *const i8, offsets: i32x4, mask: i32x4, scale: i8) -> i32x4;
3814    #[link_name = "llvm.x86.avx2.gather.d.d.256"]
3815    fn vpgatherdd(src: i32x8, slice: *const i8, offsets: i32x8, mask: i32x8, scale: i8) -> i32x8;
3816    #[link_name = "llvm.x86.avx2.gather.d.q"]
3817    fn pgatherdq(src: i64x2, slice: *const i8, offsets: i32x4, mask: i64x2, scale: i8) -> i64x2;
3818    #[link_name = "llvm.x86.avx2.gather.d.q.256"]
3819    fn vpgatherdq(src: i64x4, slice: *const i8, offsets: i32x4, mask: i64x4, scale: i8) -> i64x4;
3820    #[link_name = "llvm.x86.avx2.gather.q.d"]
3821    fn pgatherqd(src: i32x4, slice: *const i8, offsets: i64x2, mask: i32x4, scale: i8) -> i32x4;
3822    #[link_name = "llvm.x86.avx2.gather.q.d.256"]
3823    fn vpgatherqd(src: i32x4, slice: *const i8, offsets: i64x4, mask: i32x4, scale: i8) -> i32x4;
3824    #[link_name = "llvm.x86.avx2.gather.q.q"]
3825    fn pgatherqq(src: i64x2, slice: *const i8, offsets: i64x2, mask: i64x2, scale: i8) -> i64x2;
3826    #[link_name = "llvm.x86.avx2.gather.q.q.256"]
3827    fn vpgatherqq(src: i64x4, slice: *const i8, offsets: i64x4, mask: i64x4, scale: i8) -> i64x4;
3828    #[link_name = "llvm.x86.avx2.gather.d.pd"]
3829    fn pgatherdpd(
3830        src: __m128d,
3831        slice: *const i8,
3832        offsets: i32x4,
3833        mask: __m128d,
3834        scale: i8,
3835    ) -> __m128d;
3836    #[link_name = "llvm.x86.avx2.gather.d.pd.256"]
3837    fn vpgatherdpd(
3838        src: __m256d,
3839        slice: *const i8,
3840        offsets: i32x4,
3841        mask: __m256d,
3842        scale: i8,
3843    ) -> __m256d;
3844    #[link_name = "llvm.x86.avx2.gather.q.pd"]
3845    fn pgatherqpd(
3846        src: __m128d,
3847        slice: *const i8,
3848        offsets: i64x2,
3849        mask: __m128d,
3850        scale: i8,
3851    ) -> __m128d;
3852    #[link_name = "llvm.x86.avx2.gather.q.pd.256"]
3853    fn vpgatherqpd(
3854        src: __m256d,
3855        slice: *const i8,
3856        offsets: i64x4,
3857        mask: __m256d,
3858        scale: i8,
3859    ) -> __m256d;
3860    #[link_name = "llvm.x86.avx2.gather.d.ps"]
3861    fn pgatherdps(src: __m128, slice: *const i8, offsets: i32x4, mask: __m128, scale: i8)
3862    -> __m128;
3863    #[link_name = "llvm.x86.avx2.gather.d.ps.256"]
3864    fn vpgatherdps(
3865        src: __m256,
3866        slice: *const i8,
3867        offsets: i32x8,
3868        mask: __m256,
3869        scale: i8,
3870    ) -> __m256;
3871    #[link_name = "llvm.x86.avx2.gather.q.ps"]
3872    fn pgatherqps(src: __m128, slice: *const i8, offsets: i64x2, mask: __m128, scale: i8)
3873    -> __m128;
3874    #[link_name = "llvm.x86.avx2.gather.q.ps.256"]
3875    fn vpgatherqps(
3876        src: __m128,
3877        slice: *const i8,
3878        offsets: i64x4,
3879        mask: __m128,
3880        scale: i8,
3881    ) -> __m128;
3882}
3883
3884#[cfg(test)]
3885mod tests {
3886    use crate::core_arch::assert_eq_const as assert_eq;
3887
3888    use stdarch_test::simd_test;
3889
3890    use crate::core_arch::x86::*;
3891
3892    #[simd_test(enable = "avx2")]
3893    const fn test_mm256_abs_epi32() {
3894        #[rustfmt::skip]
3895        let a = _mm256_setr_epi32(
3896            0, 1, -1, i32::MAX,
3897            i32::MIN, 100, -100, -32,
3898        );
3899        let r = _mm256_abs_epi32(a);
3900        #[rustfmt::skip]
3901        let e = _mm256_setr_epi32(
3902            0, 1, 1, i32::MAX,
3903            i32::MAX.wrapping_add(1), 100, 100, 32,
3904        );
3905        assert_eq_m256i(r, e);
3906    }
3907
3908    #[simd_test(enable = "avx2")]
3909    const fn test_mm256_abs_epi16() {
3910        #[rustfmt::skip]
3911        let a = _mm256_setr_epi16(
3912            0,  1, -1, 2, -2, 3, -3, 4,
3913            -4, 5, -5, i16::MAX, i16::MIN, 100, -100, -32,
3914        );
3915        let r = _mm256_abs_epi16(a);
3916        #[rustfmt::skip]
3917        let e = _mm256_setr_epi16(
3918            0, 1, 1, 2, 2, 3, 3, 4,
3919            4, 5, 5, i16::MAX, i16::MAX.wrapping_add(1), 100, 100, 32,
3920        );
3921        assert_eq_m256i(r, e);
3922    }
3923
3924    #[simd_test(enable = "avx2")]
3925    const fn test_mm256_abs_epi8() {
3926        #[rustfmt::skip]
3927        let a = _mm256_setr_epi8(
3928            0, 1, -1, 2, -2, 3, -3, 4,
3929            -4, 5, -5, i8::MAX, i8::MIN, 100, -100, -32,
3930            0, 1, -1, 2, -2, 3, -3, 4,
3931            -4, 5, -5, i8::MAX, i8::MIN, 100, -100, -32,
3932        );
3933        let r = _mm256_abs_epi8(a);
3934        #[rustfmt::skip]
3935        let e = _mm256_setr_epi8(
3936            0, 1, 1, 2, 2, 3, 3, 4,
3937            4, 5, 5, i8::MAX, i8::MAX.wrapping_add(1), 100, 100, 32,
3938            0, 1, 1, 2, 2, 3, 3, 4,
3939            4, 5, 5, i8::MAX, i8::MAX.wrapping_add(1), 100, 100, 32,
3940        );
3941        assert_eq_m256i(r, e);
3942    }
3943
3944    #[simd_test(enable = "avx2")]
3945    const fn test_mm256_add_epi64() {
3946        let a = _mm256_setr_epi64x(-10, 0, 100, 1_000_000_000);
3947        let b = _mm256_setr_epi64x(-1, 0, 1, 2);
3948        let r = _mm256_add_epi64(a, b);
3949        let e = _mm256_setr_epi64x(-11, 0, 101, 1_000_000_002);
3950        assert_eq_m256i(r, e);
3951    }
3952
3953    #[simd_test(enable = "avx2")]
3954    const fn test_mm256_add_epi32() {
3955        let a = _mm256_setr_epi32(-1, 0, 1, 2, 3, 4, 5, 6);
3956        let b = _mm256_setr_epi32(1, 2, 3, 4, 5, 6, 7, 8);
3957        let r = _mm256_add_epi32(a, b);
3958        let e = _mm256_setr_epi32(0, 2, 4, 6, 8, 10, 12, 14);
3959        assert_eq_m256i(r, e);
3960    }
3961
3962    #[simd_test(enable = "avx2")]
3963    const fn test_mm256_add_epi16() {
3964        #[rustfmt::skip]
3965        let a = _mm256_setr_epi16(
3966            0, 1, 2, 3, 4, 5, 6, 7,
3967            8, 9, 10, 11, 12, 13, 14, 15,
3968        );
3969        #[rustfmt::skip]
3970        let b = _mm256_setr_epi16(
3971            0, 1, 2, 3, 4, 5, 6, 7,
3972            8, 9, 10, 11, 12, 13, 14, 15,
3973        );
3974        let r = _mm256_add_epi16(a, b);
3975        #[rustfmt::skip]
3976        let e = _mm256_setr_epi16(
3977            0, 2, 4, 6, 8, 10, 12, 14,
3978            16, 18, 20, 22, 24, 26, 28, 30,
3979        );
3980        assert_eq_m256i(r, e);
3981    }
3982
3983    #[simd_test(enable = "avx2")]
3984    const fn test_mm256_add_epi8() {
3985        #[rustfmt::skip]
3986        let a = _mm256_setr_epi8(
3987            0, 1, 2, 3, 4, 5, 6, 7,
3988            8, 9, 10, 11, 12, 13, 14, 15,
3989            16, 17, 18, 19, 20, 21, 22, 23,
3990            24, 25, 26, 27, 28, 29, 30, 31,
3991        );
3992        #[rustfmt::skip]
3993        let b = _mm256_setr_epi8(
3994            0, 1, 2, 3, 4, 5, 6, 7,
3995            8, 9, 10, 11, 12, 13, 14, 15,
3996            16, 17, 18, 19, 20, 21, 22, 23,
3997            24, 25, 26, 27, 28, 29, 30, 31,
3998        );
3999        let r = _mm256_add_epi8(a, b);
4000        #[rustfmt::skip]
4001        let e = _mm256_setr_epi8(
4002            0, 2, 4, 6, 8, 10, 12, 14,
4003            16, 18, 20, 22, 24, 26, 28, 30,
4004            32, 34, 36, 38, 40, 42, 44, 46,
4005            48, 50, 52, 54, 56, 58, 60, 62,
4006        );
4007        assert_eq_m256i(r, e);
4008    }
4009
4010    #[simd_test(enable = "avx2")]
4011    const fn test_mm256_adds_epi8() {
4012        #[rustfmt::skip]
4013        let a = _mm256_setr_epi8(
4014            0, 1, 2, 3, 4, 5, 6, 7,
4015            8, 9, 10, 11, 12, 13, 14, 15,
4016            16, 17, 18, 19, 20, 21, 22, 23,
4017            24, 25, 26, 27, 28, 29, 30, 31,
4018        );
4019        #[rustfmt::skip]
4020        let b = _mm256_setr_epi8(
4021            32, 33, 34, 35, 36, 37, 38, 39,
4022            40, 41, 42, 43, 44, 45, 46, 47,
4023            48, 49, 50, 51, 52, 53, 54, 55,
4024            56, 57, 58, 59, 60, 61, 62, 63,
4025        );
4026        let r = _mm256_adds_epi8(a, b);
4027        #[rustfmt::skip]
4028        let e = _mm256_setr_epi8(
4029            32, 34, 36, 38, 40, 42, 44, 46,
4030            48, 50, 52, 54, 56, 58, 60, 62,
4031            64, 66, 68, 70, 72, 74, 76, 78,
4032            80, 82, 84, 86, 88, 90, 92, 94,
4033        );
4034        assert_eq_m256i(r, e);
4035    }
4036
4037    #[simd_test(enable = "avx2")]
4038    fn test_mm256_adds_epi8_saturate_positive() {
4039        let a = _mm256_set1_epi8(0x7F);
4040        let b = _mm256_set1_epi8(1);
4041        let r = _mm256_adds_epi8(a, b);
4042        assert_eq_m256i(r, a);
4043    }
4044
4045    #[simd_test(enable = "avx2")]
4046    fn test_mm256_adds_epi8_saturate_negative() {
4047        let a = _mm256_set1_epi8(-0x80);
4048        let b = _mm256_set1_epi8(-1);
4049        let r = _mm256_adds_epi8(a, b);
4050        assert_eq_m256i(r, a);
4051    }
4052
4053    #[simd_test(enable = "avx2")]
4054    const fn test_mm256_adds_epi16() {
4055        #[rustfmt::skip]
4056        let a = _mm256_setr_epi16(
4057            0, 1, 2, 3, 4, 5, 6, 7,
4058            8, 9, 10, 11, 12, 13, 14, 15,
4059        );
4060        #[rustfmt::skip]
4061        let b = _mm256_setr_epi16(
4062            32, 33, 34, 35, 36, 37, 38, 39,
4063            40, 41, 42, 43, 44, 45, 46, 47,
4064        );
4065        let r = _mm256_adds_epi16(a, b);
4066        #[rustfmt::skip]
4067        let e = _mm256_setr_epi16(
4068            32, 34, 36, 38, 40, 42, 44, 46,
4069            48, 50, 52, 54, 56, 58, 60, 62,
4070        );
4071
4072        assert_eq_m256i(r, e);
4073    }
4074
4075    #[simd_test(enable = "avx2")]
4076    fn test_mm256_adds_epi16_saturate_positive() {
4077        let a = _mm256_set1_epi16(0x7FFF);
4078        let b = _mm256_set1_epi16(1);
4079        let r = _mm256_adds_epi16(a, b);
4080        assert_eq_m256i(r, a);
4081    }
4082
4083    #[simd_test(enable = "avx2")]
4084    fn test_mm256_adds_epi16_saturate_negative() {
4085        let a = _mm256_set1_epi16(-0x8000);
4086        let b = _mm256_set1_epi16(-1);
4087        let r = _mm256_adds_epi16(a, b);
4088        assert_eq_m256i(r, a);
4089    }
4090
4091    #[simd_test(enable = "avx2")]
4092    const fn test_mm256_adds_epu8() {
4093        #[rustfmt::skip]
4094        let a = _mm256_setr_epi8(
4095            0, 1, 2, 3, 4, 5, 6, 7,
4096            8, 9, 10, 11, 12, 13, 14, 15,
4097            16, 17, 18, 19, 20, 21, 22, 23,
4098            24, 25, 26, 27, 28, 29, 30, 31,
4099        );
4100        #[rustfmt::skip]
4101        let b = _mm256_setr_epi8(
4102            32, 33, 34, 35, 36, 37, 38, 39,
4103            40, 41, 42, 43, 44, 45, 46, 47,
4104            48, 49, 50, 51, 52, 53, 54, 55,
4105            56, 57, 58, 59, 60, 61, 62, 63,
4106        );
4107        let r = _mm256_adds_epu8(a, b);
4108        #[rustfmt::skip]
4109        let e = _mm256_setr_epi8(
4110            32, 34, 36, 38, 40, 42, 44, 46,
4111            48, 50, 52, 54, 56, 58, 60, 62,
4112            64, 66, 68, 70, 72, 74, 76, 78,
4113            80, 82, 84, 86, 88, 90, 92, 94,
4114        );
4115        assert_eq_m256i(r, e);
4116    }
4117
4118    #[simd_test(enable = "avx2")]
4119    fn test_mm256_adds_epu8_saturate() {
4120        let a = _mm256_set1_epi8(!0);
4121        let b = _mm256_set1_epi8(1);
4122        let r = _mm256_adds_epu8(a, b);
4123        assert_eq_m256i(r, a);
4124    }
4125
4126    #[simd_test(enable = "avx2")]
4127    const fn test_mm256_adds_epu16() {
4128        #[rustfmt::skip]
4129        let a = _mm256_setr_epi16(
4130            0, 1, 2, 3, 4, 5, 6, 7,
4131            8, 9, 10, 11, 12, 13, 14, 15,
4132        );
4133        #[rustfmt::skip]
4134        let b = _mm256_setr_epi16(
4135            32, 33, 34, 35, 36, 37, 38, 39,
4136            40, 41, 42, 43, 44, 45, 46, 47,
4137        );
4138        let r = _mm256_adds_epu16(a, b);
4139        #[rustfmt::skip]
4140        let e = _mm256_setr_epi16(
4141            32, 34, 36, 38, 40, 42, 44, 46,
4142            48, 50, 52, 54, 56, 58, 60, 62,
4143        );
4144
4145        assert_eq_m256i(r, e);
4146    }
4147
4148    #[simd_test(enable = "avx2")]
4149    fn test_mm256_adds_epu16_saturate() {
4150        let a = _mm256_set1_epi16(!0);
4151        let b = _mm256_set1_epi16(1);
4152        let r = _mm256_adds_epu16(a, b);
4153        assert_eq_m256i(r, a);
4154    }
4155
4156    #[simd_test(enable = "avx2")]
4157    const fn test_mm256_and_si256() {
4158        let a = _mm256_set1_epi8(5);
4159        let b = _mm256_set1_epi8(3);
4160        let got = _mm256_and_si256(a, b);
4161        assert_eq_m256i(got, _mm256_set1_epi8(1));
4162    }
4163
4164    #[simd_test(enable = "avx2")]
4165    const fn test_mm256_andnot_si256() {
4166        let a = _mm256_set1_epi8(5);
4167        let b = _mm256_set1_epi8(3);
4168        let got = _mm256_andnot_si256(a, b);
4169        assert_eq_m256i(got, _mm256_set1_epi8(2));
4170    }
4171
4172    #[simd_test(enable = "avx2")]
4173    fn test_mm256_avg_epu8() {
4174        let (a, b) = (_mm256_set1_epi8(3), _mm256_set1_epi8(9));
4175        let r = _mm256_avg_epu8(a, b);
4176        assert_eq_m256i(r, _mm256_set1_epi8(6));
4177    }
4178
4179    #[simd_test(enable = "avx2")]
4180    fn test_mm256_avg_epu16() {
4181        let (a, b) = (_mm256_set1_epi16(3), _mm256_set1_epi16(9));
4182        let r = _mm256_avg_epu16(a, b);
4183        assert_eq_m256i(r, _mm256_set1_epi16(6));
4184    }
4185
4186    #[simd_test(enable = "avx2")]
4187    const fn test_mm_blend_epi32() {
4188        let (a, b) = (_mm_set1_epi32(3), _mm_set1_epi32(9));
4189        let e = _mm_setr_epi32(9, 3, 3, 3);
4190        let r = _mm_blend_epi32::<0x01>(a, b);
4191        assert_eq_m128i(r, e);
4192
4193        let r = _mm_blend_epi32::<0x0E>(b, a);
4194        assert_eq_m128i(r, e);
4195    }
4196
4197    #[simd_test(enable = "avx2")]
4198    const fn test_mm256_blend_epi32() {
4199        let (a, b) = (_mm256_set1_epi32(3), _mm256_set1_epi32(9));
4200        let e = _mm256_setr_epi32(9, 3, 3, 3, 3, 3, 3, 3);
4201        let r = _mm256_blend_epi32::<0x01>(a, b);
4202        assert_eq_m256i(r, e);
4203
4204        let e = _mm256_setr_epi32(3, 9, 3, 3, 3, 3, 3, 9);
4205        let r = _mm256_blend_epi32::<0x82>(a, b);
4206        assert_eq_m256i(r, e);
4207
4208        let e = _mm256_setr_epi32(3, 3, 9, 9, 9, 9, 9, 3);
4209        let r = _mm256_blend_epi32::<0x7C>(a, b);
4210        assert_eq_m256i(r, e);
4211    }
4212
4213    #[simd_test(enable = "avx2")]
4214    const fn test_mm256_blend_epi16() {
4215        let (a, b) = (_mm256_set1_epi16(3), _mm256_set1_epi16(9));
4216        let e = _mm256_setr_epi16(9, 3, 3, 3, 3, 3, 3, 3, 9, 3, 3, 3, 3, 3, 3, 3);
4217        let r = _mm256_blend_epi16::<0x01>(a, b);
4218        assert_eq_m256i(r, e);
4219
4220        let r = _mm256_blend_epi16::<0xFE>(b, a);
4221        assert_eq_m256i(r, e);
4222    }
4223
4224    #[simd_test(enable = "avx2")]
4225    const fn test_mm256_blendv_epi8() {
4226        let (a, b) = (_mm256_set1_epi8(4), _mm256_set1_epi8(2));
4227        let mask = _mm256_insert_epi8::<2>(_mm256_set1_epi8(0), -1);
4228        let e = _mm256_insert_epi8::<2>(_mm256_set1_epi8(4), 2);
4229        let r = _mm256_blendv_epi8(a, b, mask);
4230        assert_eq_m256i(r, e);
4231    }
4232
4233    #[simd_test(enable = "avx2")]
4234    const fn test_mm_broadcastb_epi8() {
4235        let a = _mm_insert_epi8::<0>(_mm_set1_epi8(0x00), 0x2a);
4236        let res = _mm_broadcastb_epi8(a);
4237        assert_eq_m128i(res, _mm_set1_epi8(0x2a));
4238    }
4239
4240    #[simd_test(enable = "avx2")]
4241    const fn test_mm256_broadcastb_epi8() {
4242        let a = _mm_insert_epi8::<0>(_mm_set1_epi8(0x00), 0x2a);
4243        let res = _mm256_broadcastb_epi8(a);
4244        assert_eq_m256i(res, _mm256_set1_epi8(0x2a));
4245    }
4246
4247    #[simd_test(enable = "avx2")]
4248    const fn test_mm_broadcastd_epi32() {
4249        let a = _mm_setr_epi32(0x2a, 0x8000000, 0, 0);
4250        let res = _mm_broadcastd_epi32(a);
4251        assert_eq_m128i(res, _mm_set1_epi32(0x2a));
4252    }
4253
4254    #[simd_test(enable = "avx2")]
4255    const fn test_mm256_broadcastd_epi32() {
4256        let a = _mm_setr_epi32(0x2a, 0x8000000, 0, 0);
4257        let res = _mm256_broadcastd_epi32(a);
4258        assert_eq_m256i(res, _mm256_set1_epi32(0x2a));
4259    }
4260
4261    #[simd_test(enable = "avx2")]
4262    const fn test_mm_broadcastq_epi64() {
4263        let a = _mm_setr_epi64x(0x1ffffffff, 0);
4264        let res = _mm_broadcastq_epi64(a);
4265        assert_eq_m128i(res, _mm_set1_epi64x(0x1ffffffff));
4266    }
4267
4268    #[simd_test(enable = "avx2")]
4269    const fn test_mm256_broadcastq_epi64() {
4270        let a = _mm_setr_epi64x(0x1ffffffff, 0);
4271        let res = _mm256_broadcastq_epi64(a);
4272        assert_eq_m256i(res, _mm256_set1_epi64x(0x1ffffffff));
4273    }
4274
4275    #[simd_test(enable = "avx2")]
4276    const fn test_mm_broadcastsd_pd() {
4277        let a = _mm_setr_pd(6.88, 3.44);
4278        let res = _mm_broadcastsd_pd(a);
4279        assert_eq_m128d(res, _mm_set1_pd(6.88));
4280    }
4281
4282    #[simd_test(enable = "avx2")]
4283    const fn test_mm256_broadcastsd_pd() {
4284        let a = _mm_setr_pd(6.88, 3.44);
4285        let res = _mm256_broadcastsd_pd(a);
4286        assert_eq_m256d(res, _mm256_set1_pd(6.88f64));
4287    }
4288
4289    #[simd_test(enable = "avx2")]
4290    const fn test_mm_broadcastsi128_si256() {
4291        let a = _mm_setr_epi64x(0x0987654321012334, 0x5678909876543210);
4292        let res = _mm_broadcastsi128_si256(a);
4293        let retval = _mm256_setr_epi64x(
4294            0x0987654321012334,
4295            0x5678909876543210,
4296            0x0987654321012334,
4297            0x5678909876543210,
4298        );
4299        assert_eq_m256i(res, retval);
4300    }
4301
4302    #[simd_test(enable = "avx2")]
4303    const fn test_mm256_broadcastsi128_si256() {
4304        let a = _mm_setr_epi64x(0x0987654321012334, 0x5678909876543210);
4305        let res = _mm256_broadcastsi128_si256(a);
4306        let retval = _mm256_setr_epi64x(
4307            0x0987654321012334,
4308            0x5678909876543210,
4309            0x0987654321012334,
4310            0x5678909876543210,
4311        );
4312        assert_eq_m256i(res, retval);
4313    }
4314
4315    #[simd_test(enable = "avx2")]
4316    const fn test_mm_broadcastss_ps() {
4317        let a = _mm_setr_ps(6.88, 3.44, 0.0, 0.0);
4318        let res = _mm_broadcastss_ps(a);
4319        assert_eq_m128(res, _mm_set1_ps(6.88));
4320    }
4321
4322    #[simd_test(enable = "avx2")]
4323    const fn test_mm256_broadcastss_ps() {
4324        let a = _mm_setr_ps(6.88, 3.44, 0.0, 0.0);
4325        let res = _mm256_broadcastss_ps(a);
4326        assert_eq_m256(res, _mm256_set1_ps(6.88));
4327    }
4328
4329    #[simd_test(enable = "avx2")]
4330    const fn test_mm_broadcastw_epi16() {
4331        let a = _mm_insert_epi16::<0>(_mm_set1_epi16(0x2a), 0x22b);
4332        let res = _mm_broadcastw_epi16(a);
4333        assert_eq_m128i(res, _mm_set1_epi16(0x22b));
4334    }
4335
4336    #[simd_test(enable = "avx2")]
4337    const fn test_mm256_broadcastw_epi16() {
4338        let a = _mm_insert_epi16::<0>(_mm_set1_epi16(0x2a), 0x22b);
4339        let res = _mm256_broadcastw_epi16(a);
4340        assert_eq_m256i(res, _mm256_set1_epi16(0x22b));
4341    }
4342
4343    #[simd_test(enable = "avx2")]
4344    const fn test_mm256_cmpeq_epi8() {
4345        #[rustfmt::skip]
4346        let a = _mm256_setr_epi8(
4347            0, 1, 2, 3, 4, 5, 6, 7,
4348            8, 9, 10, 11, 12, 13, 14, 15,
4349            16, 17, 18, 19, 20, 21, 22, 23,
4350            24, 25, 26, 27, 28, 29, 30, 31,
4351        );
4352        #[rustfmt::skip]
4353        let b = _mm256_setr_epi8(
4354            31, 30, 2, 28, 27, 26, 25, 24,
4355            23, 22, 21, 20, 19, 18, 17, 16,
4356            15, 14, 13, 12, 11, 10, 9, 8,
4357            7, 6, 5, 4, 3, 2, 1, 0,
4358        );
4359        let r = _mm256_cmpeq_epi8(a, b);
4360        assert_eq_m256i(r, _mm256_insert_epi8::<2>(_mm256_set1_epi8(0), !0));
4361    }
4362
4363    #[simd_test(enable = "avx2")]
4364    const fn test_mm256_cmpeq_epi16() {
4365        #[rustfmt::skip]
4366        let a = _mm256_setr_epi16(
4367            0, 1, 2, 3, 4, 5, 6, 7,
4368            8, 9, 10, 11, 12, 13, 14, 15,
4369        );
4370        #[rustfmt::skip]
4371        let b = _mm256_setr_epi16(
4372            15, 14, 2, 12, 11, 10, 9, 8,
4373            7, 6, 5, 4, 3, 2, 1, 0,
4374        );
4375        let r = _mm256_cmpeq_epi16(a, b);
4376        assert_eq_m256i(r, _mm256_insert_epi16::<2>(_mm256_set1_epi16(0), !0));
4377    }
4378
4379    #[simd_test(enable = "avx2")]
4380    const fn test_mm256_cmpeq_epi32() {
4381        let a = _mm256_setr_epi32(0, 1, 2, 3, 4, 5, 6, 7);
4382        let b = _mm256_setr_epi32(7, 6, 2, 4, 3, 2, 1, 0);
4383        let r = _mm256_cmpeq_epi32(a, b);
4384        let e = _mm256_set1_epi32(0);
4385        let e = _mm256_insert_epi32::<2>(e, !0);
4386        assert_eq_m256i(r, e);
4387    }
4388
4389    #[simd_test(enable = "avx2")]
4390    const fn test_mm256_cmpeq_epi64() {
4391        let a = _mm256_setr_epi64x(0, 1, 2, 3);
4392        let b = _mm256_setr_epi64x(3, 2, 2, 0);
4393        let r = _mm256_cmpeq_epi64(a, b);
4394        assert_eq_m256i(r, _mm256_insert_epi64::<2>(_mm256_set1_epi64x(0), !0));
4395    }
4396
4397    #[simd_test(enable = "avx2")]
4398    const fn test_mm256_cmpgt_epi8() {
4399        let a = _mm256_insert_epi8::<0>(_mm256_set1_epi8(0), 5);
4400        let b = _mm256_set1_epi8(0);
4401        let r = _mm256_cmpgt_epi8(a, b);
4402        assert_eq_m256i(r, _mm256_insert_epi8::<0>(_mm256_set1_epi8(0), !0));
4403    }
4404
4405    #[simd_test(enable = "avx2")]
4406    const fn test_mm256_cmpgt_epi16() {
4407        let a = _mm256_insert_epi16::<0>(_mm256_set1_epi16(0), 5);
4408        let b = _mm256_set1_epi16(0);
4409        let r = _mm256_cmpgt_epi16(a, b);
4410        assert_eq_m256i(r, _mm256_insert_epi16::<0>(_mm256_set1_epi16(0), !0));
4411    }
4412
4413    #[simd_test(enable = "avx2")]
4414    const fn test_mm256_cmpgt_epi32() {
4415        let a = _mm256_insert_epi32::<0>(_mm256_set1_epi32(0), 5);
4416        let b = _mm256_set1_epi32(0);
4417        let r = _mm256_cmpgt_epi32(a, b);
4418        assert_eq_m256i(r, _mm256_insert_epi32::<0>(_mm256_set1_epi32(0), !0));
4419    }
4420
4421    #[simd_test(enable = "avx2")]
4422    const fn test_mm256_cmpgt_epi64() {
4423        let a = _mm256_insert_epi64::<0>(_mm256_set1_epi64x(0), 5);
4424        let b = _mm256_set1_epi64x(0);
4425        let r = _mm256_cmpgt_epi64(a, b);
4426        assert_eq_m256i(r, _mm256_insert_epi64::<0>(_mm256_set1_epi64x(0), !0));
4427    }
4428
4429    #[simd_test(enable = "avx2")]
4430    const fn test_mm256_cvtepi8_epi16() {
4431        #[rustfmt::skip]
4432        let a = _mm_setr_epi8(
4433            0, 0, -1, 1, -2, 2, -3, 3,
4434            -4, 4, -5, 5, -6, 6, -7, 7,
4435        );
4436        #[rustfmt::skip]
4437        let r = _mm256_setr_epi16(
4438            0, 0, -1, 1, -2, 2, -3, 3,
4439            -4, 4, -5, 5, -6, 6, -7, 7,
4440        );
4441        assert_eq_m256i(r, _mm256_cvtepi8_epi16(a));
4442    }
4443
4444    #[simd_test(enable = "avx2")]
4445    const fn test_mm256_cvtepi8_epi32() {
4446        #[rustfmt::skip]
4447        let a = _mm_setr_epi8(
4448            0, 0, -1, 1, -2, 2, -3, 3,
4449            -4, 4, -5, 5, -6, 6, -7, 7,
4450        );
4451        let r = _mm256_setr_epi32(0, 0, -1, 1, -2, 2, -3, 3);
4452        assert_eq_m256i(r, _mm256_cvtepi8_epi32(a));
4453    }
4454
4455    #[simd_test(enable = "avx2")]
4456    const fn test_mm256_cvtepi8_epi64() {
4457        #[rustfmt::skip]
4458        let a = _mm_setr_epi8(
4459            0, 0, -1, 1, -2, 2, -3, 3,
4460            -4, 4, -5, 5, -6, 6, -7, 7,
4461        );
4462        let r = _mm256_setr_epi64x(0, 0, -1, 1);
4463        assert_eq_m256i(r, _mm256_cvtepi8_epi64(a));
4464    }
4465
4466    #[simd_test(enable = "avx2")]
4467    const fn test_mm256_cvtepi16_epi32() {
4468        let a = _mm_setr_epi16(0, 0, -1, 1, -2, 2, -3, 3);
4469        let r = _mm256_setr_epi32(0, 0, -1, 1, -2, 2, -3, 3);
4470        assert_eq_m256i(r, _mm256_cvtepi16_epi32(a));
4471    }
4472
4473    #[simd_test(enable = "avx2")]
4474    const fn test_mm256_cvtepi16_epi64() {
4475        let a = _mm_setr_epi16(0, 0, -1, 1, -2, 2, -3, 3);
4476        let r = _mm256_setr_epi64x(0, 0, -1, 1);
4477        assert_eq_m256i(r, _mm256_cvtepi16_epi64(a));
4478    }
4479
4480    #[simd_test(enable = "avx2")]
4481    const fn test_mm256_cvtepi32_epi64() {
4482        let a = _mm_setr_epi32(0, 0, -1, 1);
4483        let r = _mm256_setr_epi64x(0, 0, -1, 1);
4484        assert_eq_m256i(r, _mm256_cvtepi32_epi64(a));
4485    }
4486
4487    #[simd_test(enable = "avx2")]
4488    const fn test_mm256_cvtepu16_epi32() {
4489        let a = _mm_setr_epi16(0, 1, 2, 3, 4, 5, 6, 7);
4490        let r = _mm256_setr_epi32(0, 1, 2, 3, 4, 5, 6, 7);
4491        assert_eq_m256i(r, _mm256_cvtepu16_epi32(a));
4492    }
4493
4494    #[simd_test(enable = "avx2")]
4495    const fn test_mm256_cvtepu16_epi64() {
4496        let a = _mm_setr_epi16(0, 1, 2, 3, 4, 5, 6, 7);
4497        let r = _mm256_setr_epi64x(0, 1, 2, 3);
4498        assert_eq_m256i(r, _mm256_cvtepu16_epi64(a));
4499    }
4500
4501    #[simd_test(enable = "avx2")]
4502    const fn test_mm256_cvtepu32_epi64() {
4503        let a = _mm_setr_epi32(0, 1, 2, 3);
4504        let r = _mm256_setr_epi64x(0, 1, 2, 3);
4505        assert_eq_m256i(r, _mm256_cvtepu32_epi64(a));
4506    }
4507
4508    #[simd_test(enable = "avx2")]
4509    const fn test_mm256_cvtepu8_epi16() {
4510        #[rustfmt::skip]
4511        let a = _mm_setr_epi8(
4512            0, 1, 2, 3, 4, 5, 6, 7,
4513            8, 9, 10, 11, 12, 13, 14, 15,
4514        );
4515        #[rustfmt::skip]
4516        let r = _mm256_setr_epi16(
4517            0, 1, 2, 3, 4, 5, 6, 7,
4518            8, 9, 10, 11, 12, 13, 14, 15,
4519        );
4520        assert_eq_m256i(r, _mm256_cvtepu8_epi16(a));
4521    }
4522
4523    #[simd_test(enable = "avx2")]
4524    const fn test_mm256_cvtepu8_epi32() {
4525        #[rustfmt::skip]
4526        let a = _mm_setr_epi8(
4527            0, 1, 2, 3, 4, 5, 6, 7,
4528            8, 9, 10, 11, 12, 13, 14, 15,
4529        );
4530        let r = _mm256_setr_epi32(0, 1, 2, 3, 4, 5, 6, 7);
4531        assert_eq_m256i(r, _mm256_cvtepu8_epi32(a));
4532    }
4533
4534    #[simd_test(enable = "avx2")]
4535    const fn test_mm256_cvtepu8_epi64() {
4536        #[rustfmt::skip]
4537        let a = _mm_setr_epi8(
4538            0, 1, 2, 3, 4, 5, 6, 7,
4539            8, 9, 10, 11, 12, 13, 14, 15,
4540        );
4541        let r = _mm256_setr_epi64x(0, 1, 2, 3);
4542        assert_eq_m256i(r, _mm256_cvtepu8_epi64(a));
4543    }
4544
4545    #[simd_test(enable = "avx2")]
4546    const fn test_mm256_extracti128_si256() {
4547        let a = _mm256_setr_epi64x(1, 2, 3, 4);
4548        let r = _mm256_extracti128_si256::<1>(a);
4549        let e = _mm_setr_epi64x(3, 4);
4550        assert_eq_m128i(r, e);
4551    }
4552
4553    #[simd_test(enable = "avx2")]
4554    const fn test_mm256_hadd_epi16() {
4555        let a = _mm256_set1_epi16(2);
4556        let b = _mm256_set1_epi16(4);
4557        let r = _mm256_hadd_epi16(a, b);
4558        let e = _mm256_setr_epi16(4, 4, 4, 4, 8, 8, 8, 8, 4, 4, 4, 4, 8, 8, 8, 8);
4559        assert_eq_m256i(r, e);
4560    }
4561
4562    #[simd_test(enable = "avx2")]
4563    const fn test_mm256_hadd_epi32() {
4564        let a = _mm256_set1_epi32(2);
4565        let b = _mm256_set1_epi32(4);
4566        let r = _mm256_hadd_epi32(a, b);
4567        let e = _mm256_setr_epi32(4, 4, 8, 8, 4, 4, 8, 8);
4568        assert_eq_m256i(r, e);
4569    }
4570
4571    #[simd_test(enable = "avx2")]
4572    fn test_mm256_hadds_epi16() {
4573        let a = _mm256_set1_epi16(2);
4574        let a = _mm256_insert_epi16::<0>(a, 0x7fff);
4575        let a = _mm256_insert_epi16::<1>(a, 1);
4576        let b = _mm256_set1_epi16(4);
4577        let r = _mm256_hadds_epi16(a, b);
4578        #[rustfmt::skip]
4579        let e = _mm256_setr_epi16(
4580            0x7FFF, 4, 4, 4, 8, 8, 8, 8,
4581            4, 4, 4, 4, 8, 8, 8, 8,
4582        );
4583        assert_eq_m256i(r, e);
4584    }
4585
4586    #[simd_test(enable = "avx2")]
4587    const fn test_mm256_hsub_epi16() {
4588        let a = _mm256_set1_epi16(2);
4589        let b = _mm256_set1_epi16(4);
4590        let r = _mm256_hsub_epi16(a, b);
4591        let e = _mm256_set1_epi16(0);
4592        assert_eq_m256i(r, e);
4593    }
4594
4595    #[simd_test(enable = "avx2")]
4596    const fn test_mm256_hsub_epi32() {
4597        let a = _mm256_set1_epi32(2);
4598        let b = _mm256_set1_epi32(4);
4599        let r = _mm256_hsub_epi32(a, b);
4600        let e = _mm256_set1_epi32(0);
4601        assert_eq_m256i(r, e);
4602    }
4603
4604    #[simd_test(enable = "avx2")]
4605    fn test_mm256_hsubs_epi16() {
4606        let a = _mm256_set1_epi16(2);
4607        let a = _mm256_insert_epi16::<0>(a, 0x7fff);
4608        let a = _mm256_insert_epi16::<1>(a, -1);
4609        let b = _mm256_set1_epi16(4);
4610        let r = _mm256_hsubs_epi16(a, b);
4611        let e = _mm256_insert_epi16::<0>(_mm256_set1_epi16(0), 0x7FFF);
4612        assert_eq_m256i(r, e);
4613    }
4614
4615    #[simd_test(enable = "avx2")]
4616    fn test_mm256_madd_epi16() {
4617        let a = _mm256_set1_epi16(2);
4618        let b = _mm256_set1_epi16(4);
4619        let r = _mm256_madd_epi16(a, b);
4620        let e = _mm256_set1_epi32(16);
4621        assert_eq_m256i(r, e);
4622    }
4623
4624    #[target_feature(enable = "avx2")]
4625    #[cfg_attr(test, assert_instr(vpmaddwd))]
4626    unsafe fn test_mm256_madd_epi16_mul_one(v: __m256i) -> __m256i {
4627        // This is a trick used in the adler32 algorithm to get a widening addition. The
4628        // multiplication by 1 is trivial, but must not be optimized out because then the vpmaddwd
4629        // instruction is no longer selected. The assert_instr verifies that this is the case.
4630        let one_v = _mm256_set1_epi16(1);
4631        _mm256_madd_epi16(v, one_v)
4632    }
4633
4634    #[target_feature(enable = "avx2")]
4635    #[cfg_attr(test, assert_instr(vpmaddwd))]
4636    unsafe fn test_mm256_madd_epi16_shl(v: __m256i) -> __m256i {
4637        // This is a trick used in the base64 algorithm to get a widening addition. Instead of a
4638        // multiplication, a vector shl is used. In LLVM 22 that breaks the pattern recognition
4639        // for the automatic optimization to vpmaddwd.
4640        let shift_value = _mm256_set1_epi32(12i32);
4641        _mm256_madd_epi16(v, shift_value)
4642    }
4643
4644    #[simd_test(enable = "avx2")]
4645    const fn test_mm256_inserti128_si256() {
4646        let a = _mm256_setr_epi64x(1, 2, 3, 4);
4647        let b = _mm_setr_epi64x(7, 8);
4648        let r = _mm256_inserti128_si256::<1>(a, b);
4649        let e = _mm256_setr_epi64x(1, 2, 7, 8);
4650        assert_eq_m256i(r, e);
4651    }
4652
4653    #[simd_test(enable = "avx2")]
4654    fn test_mm256_maddubs_epi16() {
4655        let a = _mm256_set1_epi8(2);
4656        let b = _mm256_set1_epi8(4);
4657        let r = _mm256_maddubs_epi16(a, b);
4658        let e = _mm256_set1_epi16(16);
4659        assert_eq_m256i(r, e);
4660    }
4661
4662    #[simd_test(enable = "avx2")]
4663    const fn test_mm_maskload_epi32() {
4664        let nums = [1, 2, 3, 4];
4665        let a = &nums as *const i32;
4666        let mask = _mm_setr_epi32(-1, 0, 0, -1);
4667        let r = unsafe { _mm_maskload_epi32(a, mask) };
4668        let e = _mm_setr_epi32(1, 0, 0, 4);
4669        assert_eq_m128i(r, e);
4670    }
4671
4672    #[simd_test(enable = "avx2")]
4673    const fn test_mm256_maskload_epi32() {
4674        let nums = [1, 2, 3, 4, 5, 6, 7, 8];
4675        let a = &nums as *const i32;
4676        let mask = _mm256_setr_epi32(-1, 0, 0, -1, 0, -1, -1, 0);
4677        let r = unsafe { _mm256_maskload_epi32(a, mask) };
4678        let e = _mm256_setr_epi32(1, 0, 0, 4, 0, 6, 7, 0);
4679        assert_eq_m256i(r, e);
4680    }
4681
4682    #[simd_test(enable = "avx2")]
4683    const fn test_mm_maskload_epi64() {
4684        let nums = [1_i64, 2_i64];
4685        let a = &nums as *const i64;
4686        let mask = _mm_setr_epi64x(0, -1);
4687        let r = unsafe { _mm_maskload_epi64(a, mask) };
4688        let e = _mm_setr_epi64x(0, 2);
4689        assert_eq_m128i(r, e);
4690    }
4691
4692    #[simd_test(enable = "avx2")]
4693    const fn test_mm256_maskload_epi64() {
4694        let nums = [1_i64, 2_i64, 3_i64, 4_i64];
4695        let a = &nums as *const i64;
4696        let mask = _mm256_setr_epi64x(0, -1, -1, 0);
4697        let r = unsafe { _mm256_maskload_epi64(a, mask) };
4698        let e = _mm256_setr_epi64x(0, 2, 3, 0);
4699        assert_eq_m256i(r, e);
4700    }
4701
4702    #[simd_test(enable = "avx2")]
4703    const fn test_mm_maskstore_epi32() {
4704        let a = _mm_setr_epi32(1, 2, 3, 4);
4705        let mut arr = [-1, -1, -1, -1];
4706        let mask = _mm_setr_epi32(-1, 0, 0, -1);
4707        unsafe {
4708            _mm_maskstore_epi32(arr.as_mut_ptr(), mask, a);
4709        }
4710        let e = [1, -1, -1, 4];
4711        assert_eq!(arr, e);
4712    }
4713
4714    #[simd_test(enable = "avx2")]
4715    const fn test_mm256_maskstore_epi32() {
4716        let a = _mm256_setr_epi32(1, 0x6d726f, 3, 42, 0x777161, 6, 7, 8);
4717        let mut arr = [-1, -1, -1, 0x776173, -1, 0x68657265, -1, -1];
4718        let mask = _mm256_setr_epi32(-1, 0, 0, -1, 0, -1, -1, 0);
4719        unsafe {
4720            _mm256_maskstore_epi32(arr.as_mut_ptr(), mask, a);
4721        }
4722        let e = [1, -1, -1, 42, -1, 6, 7, -1];
4723        assert_eq!(arr, e);
4724    }
4725
4726    #[simd_test(enable = "avx2")]
4727    const fn test_mm_maskstore_epi64() {
4728        let a = _mm_setr_epi64x(1_i64, 2_i64);
4729        let mut arr = [-1_i64, -1_i64];
4730        let mask = _mm_setr_epi64x(0, -1);
4731        unsafe {
4732            _mm_maskstore_epi64(arr.as_mut_ptr(), mask, a);
4733        }
4734        let e = [-1, 2];
4735        assert_eq!(arr, e);
4736    }
4737
4738    #[simd_test(enable = "avx2")]
4739    const fn test_mm256_maskstore_epi64() {
4740        let a = _mm256_setr_epi64x(1_i64, 2_i64, 3_i64, 4_i64);
4741        let mut arr = [-1_i64, -1_i64, -1_i64, -1_i64];
4742        let mask = _mm256_setr_epi64x(0, -1, -1, 0);
4743        unsafe {
4744            _mm256_maskstore_epi64(arr.as_mut_ptr(), mask, a);
4745        }
4746        let e = [-1, 2, 3, -1];
4747        assert_eq!(arr, e);
4748    }
4749
4750    #[simd_test(enable = "avx2")]
4751    const fn test_mm256_max_epi16() {
4752        let a = _mm256_set1_epi16(2);
4753        let b = _mm256_set1_epi16(4);
4754        let r = _mm256_max_epi16(a, b);
4755        assert_eq_m256i(r, b);
4756    }
4757
4758    #[simd_test(enable = "avx2")]
4759    const fn test_mm256_max_epi32() {
4760        let a = _mm256_set1_epi32(2);
4761        let b = _mm256_set1_epi32(4);
4762        let r = _mm256_max_epi32(a, b);
4763        assert_eq_m256i(r, b);
4764    }
4765
4766    #[simd_test(enable = "avx2")]
4767    const fn test_mm256_max_epi8() {
4768        let a = _mm256_set1_epi8(2);
4769        let b = _mm256_set1_epi8(4);
4770        let r = _mm256_max_epi8(a, b);
4771        assert_eq_m256i(r, b);
4772    }
4773
4774    #[simd_test(enable = "avx2")]
4775    const fn test_mm256_max_epu16() {
4776        let a = _mm256_set1_epi16(2);
4777        let b = _mm256_set1_epi16(4);
4778        let r = _mm256_max_epu16(a, b);
4779        assert_eq_m256i(r, b);
4780    }
4781
4782    #[simd_test(enable = "avx2")]
4783    const fn test_mm256_max_epu32() {
4784        let a = _mm256_set1_epi32(2);
4785        let b = _mm256_set1_epi32(4);
4786        let r = _mm256_max_epu32(a, b);
4787        assert_eq_m256i(r, b);
4788    }
4789
4790    #[simd_test(enable = "avx2")]
4791    const fn test_mm256_max_epu8() {
4792        let a = _mm256_set1_epi8(2);
4793        let b = _mm256_set1_epi8(4);
4794        let r = _mm256_max_epu8(a, b);
4795        assert_eq_m256i(r, b);
4796    }
4797
4798    #[simd_test(enable = "avx2")]
4799    const fn test_mm256_min_epi16() {
4800        let a = _mm256_set1_epi16(2);
4801        let b = _mm256_set1_epi16(4);
4802        let r = _mm256_min_epi16(a, b);
4803        assert_eq_m256i(r, a);
4804    }
4805
4806    #[simd_test(enable = "avx2")]
4807    const fn test_mm256_min_epi32() {
4808        let a = _mm256_set1_epi32(2);
4809        let b = _mm256_set1_epi32(4);
4810        let r = _mm256_min_epi32(a, b);
4811        assert_eq_m256i(r, a);
4812    }
4813
4814    #[simd_test(enable = "avx2")]
4815    const fn test_mm256_min_epi8() {
4816        let a = _mm256_set1_epi8(2);
4817        let b = _mm256_set1_epi8(4);
4818        let r = _mm256_min_epi8(a, b);
4819        assert_eq_m256i(r, a);
4820    }
4821
4822    #[simd_test(enable = "avx2")]
4823    const fn test_mm256_min_epu16() {
4824        let a = _mm256_set1_epi16(2);
4825        let b = _mm256_set1_epi16(4);
4826        let r = _mm256_min_epu16(a, b);
4827        assert_eq_m256i(r, a);
4828    }
4829
4830    #[simd_test(enable = "avx2")]
4831    const fn test_mm256_min_epu32() {
4832        let a = _mm256_set1_epi32(2);
4833        let b = _mm256_set1_epi32(4);
4834        let r = _mm256_min_epu32(a, b);
4835        assert_eq_m256i(r, a);
4836    }
4837
4838    #[simd_test(enable = "avx2")]
4839    const fn test_mm256_min_epu8() {
4840        let a = _mm256_set1_epi8(2);
4841        let b = _mm256_set1_epi8(4);
4842        let r = _mm256_min_epu8(a, b);
4843        assert_eq_m256i(r, a);
4844    }
4845
4846    #[simd_test(enable = "avx2")]
4847    const fn test_mm256_movemask_epi8() {
4848        let a = _mm256_set1_epi8(-1);
4849        let r = _mm256_movemask_epi8(a);
4850        let e = -1;
4851        assert_eq!(r, e);
4852    }
4853
4854    #[simd_test(enable = "avx2")]
4855    fn test_mm256_mpsadbw_epu8() {
4856        let a = _mm256_set1_epi8(2);
4857        let b = _mm256_set1_epi8(4);
4858        let r = _mm256_mpsadbw_epu8::<0>(a, b);
4859        let e = _mm256_set1_epi16(8);
4860        assert_eq_m256i(r, e);
4861    }
4862
4863    #[simd_test(enable = "avx2")]
4864    const fn test_mm256_mul_epi32() {
4865        let a = _mm256_setr_epi32(0, 0, 0, 0, 2, 2, 2, 2);
4866        let b = _mm256_setr_epi32(1, 2, 3, 4, 5, 6, 7, 8);
4867        let r = _mm256_mul_epi32(a, b);
4868        let e = _mm256_setr_epi64x(0, 0, 10, 14);
4869        assert_eq_m256i(r, e);
4870    }
4871
4872    #[simd_test(enable = "avx2")]
4873    const fn test_mm256_mul_epu32() {
4874        let a = _mm256_setr_epi32(0, 0, 0, 0, 2, 2, 2, 2);
4875        let b = _mm256_setr_epi32(1, 2, 3, 4, 5, 6, 7, 8);
4876        let r = _mm256_mul_epu32(a, b);
4877        let e = _mm256_setr_epi64x(0, 0, 10, 14);
4878        assert_eq_m256i(r, e);
4879    }
4880
4881    #[simd_test(enable = "avx2")]
4882    fn test_mm256_mulhi_epi16() {
4883        let a = _mm256_set1_epi16(6535);
4884        let b = _mm256_set1_epi16(6535);
4885        let r = _mm256_mulhi_epi16(a, b);
4886        let e = _mm256_set1_epi16(651);
4887        assert_eq_m256i(r, e);
4888    }
4889
4890    #[simd_test(enable = "avx2")]
4891    fn test_mm256_mulhi_epu16() {
4892        let a = _mm256_set1_epi16(6535);
4893        let b = _mm256_set1_epi16(6535);
4894        let r = _mm256_mulhi_epu16(a, b);
4895        let e = _mm256_set1_epi16(651);
4896        assert_eq_m256i(r, e);
4897    }
4898
4899    #[simd_test(enable = "avx2")]
4900    const fn test_mm256_mullo_epi16() {
4901        let a = _mm256_set1_epi16(2);
4902        let b = _mm256_set1_epi16(4);
4903        let r = _mm256_mullo_epi16(a, b);
4904        let e = _mm256_set1_epi16(8);
4905        assert_eq_m256i(r, e);
4906    }
4907
4908    #[simd_test(enable = "avx2")]
4909    const fn test_mm256_mullo_epi32() {
4910        let a = _mm256_set1_epi32(2);
4911        let b = _mm256_set1_epi32(4);
4912        let r = _mm256_mullo_epi32(a, b);
4913        let e = _mm256_set1_epi32(8);
4914        assert_eq_m256i(r, e);
4915    }
4916
4917    #[simd_test(enable = "avx2")]
4918    fn test_mm256_mulhrs_epi16() {
4919        let a = _mm256_set1_epi16(2);
4920        let b = _mm256_set1_epi16(4);
4921        let r = _mm256_mullo_epi16(a, b);
4922        let e = _mm256_set1_epi16(8);
4923        assert_eq_m256i(r, e);
4924    }
4925
4926    #[simd_test(enable = "avx2")]
4927    const fn test_mm256_or_si256() {
4928        let a = _mm256_set1_epi8(-1);
4929        let b = _mm256_set1_epi8(0);
4930        let r = _mm256_or_si256(a, b);
4931        assert_eq_m256i(r, a);
4932    }
4933
4934    #[simd_test(enable = "avx2")]
4935    fn test_mm256_packs_epi16() {
4936        let a = _mm256_set1_epi16(2);
4937        let b = _mm256_set1_epi16(4);
4938        let r = _mm256_packs_epi16(a, b);
4939        #[rustfmt::skip]
4940        let e = _mm256_setr_epi8(
4941            2, 2, 2, 2, 2, 2, 2, 2,
4942            4, 4, 4, 4, 4, 4, 4, 4,
4943            2, 2, 2, 2, 2, 2, 2, 2,
4944            4, 4, 4, 4, 4, 4, 4, 4,
4945        );
4946
4947        assert_eq_m256i(r, e);
4948    }
4949
4950    #[simd_test(enable = "avx2")]
4951    fn test_mm256_packs_epi32() {
4952        let a = _mm256_set1_epi32(2);
4953        let b = _mm256_set1_epi32(4);
4954        let r = _mm256_packs_epi32(a, b);
4955        let e = _mm256_setr_epi16(2, 2, 2, 2, 4, 4, 4, 4, 2, 2, 2, 2, 4, 4, 4, 4);
4956
4957        assert_eq_m256i(r, e);
4958    }
4959
4960    #[simd_test(enable = "avx2")]
4961    fn test_mm256_packus_epi16() {
4962        let a = _mm256_set1_epi16(2);
4963        let b = _mm256_set1_epi16(4);
4964        let r = _mm256_packus_epi16(a, b);
4965        #[rustfmt::skip]
4966        let e = _mm256_setr_epi8(
4967            2, 2, 2, 2, 2, 2, 2, 2,
4968            4, 4, 4, 4, 4, 4, 4, 4,
4969            2, 2, 2, 2, 2, 2, 2, 2,
4970            4, 4, 4, 4, 4, 4, 4, 4,
4971        );
4972
4973        assert_eq_m256i(r, e);
4974    }
4975
4976    #[simd_test(enable = "avx2")]
4977    fn test_mm256_packus_epi32() {
4978        let a = _mm256_set1_epi32(2);
4979        let b = _mm256_set1_epi32(4);
4980        let r = _mm256_packus_epi32(a, b);
4981        let e = _mm256_setr_epi16(2, 2, 2, 2, 4, 4, 4, 4, 2, 2, 2, 2, 4, 4, 4, 4);
4982
4983        assert_eq_m256i(r, e);
4984    }
4985
4986    #[simd_test(enable = "avx2")]
4987    fn test_mm256_sad_epu8() {
4988        let a = _mm256_set1_epi8(2);
4989        let b = _mm256_set1_epi8(4);
4990        let r = _mm256_sad_epu8(a, b);
4991        let e = _mm256_set1_epi64x(16);
4992        assert_eq_m256i(r, e);
4993    }
4994
4995    #[simd_test(enable = "avx2")]
4996    const fn test_mm256_shufflehi_epi16() {
4997        #[rustfmt::skip]
4998        let a = _mm256_setr_epi16(
4999            0, 1, 2, 3, 11, 22, 33, 44,
5000            4, 5, 6, 7, 55, 66, 77, 88,
5001        );
5002        #[rustfmt::skip]
5003        let e = _mm256_setr_epi16(
5004            0, 1, 2, 3, 44, 22, 22, 11,
5005            4, 5, 6, 7, 88, 66, 66, 55,
5006        );
5007        let r = _mm256_shufflehi_epi16::<0b00_01_01_11>(a);
5008        assert_eq_m256i(r, e);
5009    }
5010
5011    #[simd_test(enable = "avx2")]
5012    const fn test_mm256_shufflelo_epi16() {
5013        #[rustfmt::skip]
5014        let a = _mm256_setr_epi16(
5015            11, 22, 33, 44, 0, 1, 2, 3,
5016            55, 66, 77, 88, 4, 5, 6, 7,
5017        );
5018        #[rustfmt::skip]
5019        let e = _mm256_setr_epi16(
5020            44, 22, 22, 11, 0, 1, 2, 3,
5021            88, 66, 66, 55, 4, 5, 6, 7,
5022        );
5023        let r = _mm256_shufflelo_epi16::<0b00_01_01_11>(a);
5024        assert_eq_m256i(r, e);
5025    }
5026
5027    #[simd_test(enable = "avx2")]
5028    fn test_mm256_sign_epi16() {
5029        let a = _mm256_set1_epi16(2);
5030        let b = _mm256_set1_epi16(-1);
5031        let r = _mm256_sign_epi16(a, b);
5032        let e = _mm256_set1_epi16(-2);
5033        assert_eq_m256i(r, e);
5034    }
5035
5036    #[simd_test(enable = "avx2")]
5037    fn test_mm256_sign_epi32() {
5038        let a = _mm256_set1_epi32(2);
5039        let b = _mm256_set1_epi32(-1);
5040        let r = _mm256_sign_epi32(a, b);
5041        let e = _mm256_set1_epi32(-2);
5042        assert_eq_m256i(r, e);
5043    }
5044
5045    #[simd_test(enable = "avx2")]
5046    fn test_mm256_sign_epi8() {
5047        let a = _mm256_set1_epi8(2);
5048        let b = _mm256_set1_epi8(-1);
5049        let r = _mm256_sign_epi8(a, b);
5050        let e = _mm256_set1_epi8(-2);
5051        assert_eq_m256i(r, e);
5052    }
5053
5054    #[simd_test(enable = "avx2")]
5055    fn test_mm256_sll_epi16() {
5056        let a = _mm256_set1_epi16(0xFF);
5057        let b = _mm_insert_epi16::<0>(_mm_set1_epi16(0), 4);
5058        let r = _mm256_sll_epi16(a, b);
5059        assert_eq_m256i(r, _mm256_set1_epi16(0xFF0));
5060    }
5061
5062    #[simd_test(enable = "avx2")]
5063    fn test_mm256_sll_epi32() {
5064        let a = _mm256_set1_epi32(0xFFFF);
5065        let b = _mm_insert_epi32::<0>(_mm_set1_epi32(0), 4);
5066        let r = _mm256_sll_epi32(a, b);
5067        assert_eq_m256i(r, _mm256_set1_epi32(0xFFFF0));
5068    }
5069
5070    #[simd_test(enable = "avx2")]
5071    fn test_mm256_sll_epi64() {
5072        let a = _mm256_set1_epi64x(0xFFFFFFFF);
5073        let b = _mm_insert_epi64::<0>(_mm_set1_epi64x(0), 4);
5074        let r = _mm256_sll_epi64(a, b);
5075        assert_eq_m256i(r, _mm256_set1_epi64x(0xFFFFFFFF0));
5076    }
5077
5078    #[simd_test(enable = "avx2")]
5079    const fn test_mm256_slli_epi16() {
5080        assert_eq_m256i(
5081            _mm256_slli_epi16::<4>(_mm256_set1_epi16(0xFF)),
5082            _mm256_set1_epi16(0xFF0),
5083        );
5084    }
5085
5086    #[simd_test(enable = "avx2")]
5087    const fn test_mm256_slli_epi32() {
5088        assert_eq_m256i(
5089            _mm256_slli_epi32::<4>(_mm256_set1_epi32(0xFFFF)),
5090            _mm256_set1_epi32(0xFFFF0),
5091        );
5092    }
5093
5094    #[simd_test(enable = "avx2")]
5095    const fn test_mm256_slli_epi64() {
5096        assert_eq_m256i(
5097            _mm256_slli_epi64::<4>(_mm256_set1_epi64x(0xFFFFFFFF)),
5098            _mm256_set1_epi64x(0xFFFFFFFF0),
5099        );
5100    }
5101
5102    #[simd_test(enable = "avx2")]
5103    const fn test_mm256_slli_si256() {
5104        let a = _mm256_set1_epi64x(0xFFFFFFFF);
5105        let r = _mm256_slli_si256::<3>(a);
5106        assert_eq_m256i(r, _mm256_set1_epi64x(0xFFFFFFFF000000));
5107    }
5108
5109    #[simd_test(enable = "avx2")]
5110    fn test_mm_sllv_epi32() {
5111        let a = _mm_set1_epi32(2);
5112        let b = _mm_set1_epi32(1);
5113        let r = _mm_sllv_epi32(a, b);
5114        let e = _mm_set1_epi32(4);
5115        assert_eq_m128i(r, e);
5116    }
5117
5118    #[simd_test(enable = "avx2")]
5119    fn test_mm256_sllv_epi32() {
5120        let a = _mm256_set1_epi32(2);
5121        let b = _mm256_set1_epi32(1);
5122        let r = _mm256_sllv_epi32(a, b);
5123        let e = _mm256_set1_epi32(4);
5124        assert_eq_m256i(r, e);
5125    }
5126
5127    #[simd_test(enable = "avx2")]
5128    fn test_mm_sllv_epi64() {
5129        let a = _mm_set_epi64x(2, 3);
5130        let b = _mm_set_epi64x(1, 2);
5131        let r = _mm_sllv_epi64(a, b);
5132        // Compare with the scalar version of the same computation.
5133        let e = _mm_set_epi64x(2i64.unbounded_shl(1), 3i64.unbounded_shl(2));
5134        assert_eq_m128i(r, e);
5135        // Compare with hardcoded output.
5136        let e = _mm_set_epi64x(4, 12);
5137        assert_eq_m128i(r, e);
5138
5139        // The shift has unbounded semantics: if the shift amount
5140        // is >= the number of bits the result is 0.
5141        let a = _mm_set_epi64x(1, 2);
5142        let b = _mm_set_epi64x(64, 65);
5143        let r = _mm_sllv_epi64(a, b);
5144        let e = _mm_set_epi64x(1i64.unbounded_shl(64), 2i64.unbounded_shl(65));
5145        assert_eq_m128i(r, e);
5146        let e = _mm_set_epi64x(0, 0);
5147        assert_eq_m128i(r, e);
5148    }
5149
5150    #[simd_test(enable = "avx2")]
5151    fn test_mm256_sllv_epi64() {
5152        let a = _mm256_set1_epi64x(2);
5153        let b = _mm256_set1_epi64x(1);
5154        let r = _mm256_sllv_epi64(a, b);
5155        let e = _mm256_set1_epi64x(4);
5156        assert_eq_m256i(r, e);
5157    }
5158
5159    #[simd_test(enable = "avx2")]
5160    fn test_mm256_sra_epi16() {
5161        let a = _mm256_set1_epi16(-1);
5162        let b = _mm_setr_epi16(1, 0, 0, 0, 0, 0, 0, 0);
5163        let r = _mm256_sra_epi16(a, b);
5164        assert_eq_m256i(r, _mm256_set1_epi16(-1));
5165    }
5166
5167    #[simd_test(enable = "avx2")]
5168    fn test_mm256_sra_epi32() {
5169        let a = _mm256_set1_epi32(-1);
5170        let b = _mm_insert_epi32::<0>(_mm_set1_epi32(0), 1);
5171        let r = _mm256_sra_epi32(a, b);
5172        assert_eq_m256i(r, _mm256_set1_epi32(-1));
5173    }
5174
5175    #[simd_test(enable = "avx2")]
5176    const fn test_mm256_srai_epi16() {
5177        assert_eq_m256i(
5178            _mm256_srai_epi16::<1>(_mm256_set1_epi16(-1)),
5179            _mm256_set1_epi16(-1),
5180        );
5181    }
5182
5183    #[simd_test(enable = "avx2")]
5184    const fn test_mm256_srai_epi32() {
5185        assert_eq_m256i(
5186            _mm256_srai_epi32::<1>(_mm256_set1_epi32(-1)),
5187            _mm256_set1_epi32(-1),
5188        );
5189    }
5190
5191    #[simd_test(enable = "avx2")]
5192    fn test_mm_srav_epi32() {
5193        let a = _mm_set_epi32(16, -32, 64, -128);
5194        let b = _mm_set_epi32(4, 3, 2, 1);
5195        let r = _mm_srav_epi32(a, b);
5196        let e = _mm_set_epi32(1, -4, 16, -64);
5197        assert_eq_m128i(r, e);
5198
5199        // The shift has unbounded semantics: if the shift amount
5200        // is >= the number of bits the result is -1.
5201        let a = _mm_set_epi32(-16, -32, -64, -128);
5202        let b = _mm_set_epi32(31, 32, 33, 0);
5203        let r = _mm_srav_epi32(a, b);
5204        // Compare with the scalar version of the same computation.
5205        let e = _mm_set_epi32(
5206            (-16i32).unbounded_shr(31),
5207            (-32i32).unbounded_shr(32),
5208            (-64i32).unbounded_shr(33),
5209            (-128i32).unbounded_shr(0),
5210        );
5211        assert_eq_m128i(r, e);
5212        // Compare with hardcoded output.
5213        let e = _mm_set_epi32(-1, -1, -1, -128);
5214        assert_eq_m128i(r, e);
5215    }
5216
5217    #[simd_test(enable = "avx2")]
5218    fn test_mm256_srav_epi32() {
5219        let a = _mm256_set1_epi32(4);
5220        let count = _mm256_set1_epi32(1);
5221        let r = _mm256_srav_epi32(a, count);
5222        let e = _mm256_set1_epi32(2);
5223        assert_eq_m256i(r, e);
5224    }
5225
5226    #[simd_test(enable = "avx2")]
5227    const fn test_mm256_srli_si256() {
5228        #[rustfmt::skip]
5229        let a = _mm256_setr_epi8(
5230            1, 2, 3, 4, 5, 6, 7, 8,
5231            9, 10, 11, 12, 13, 14, 15, 16,
5232            17, 18, 19, 20, 21, 22, 23, 24,
5233            25, 26, 27, 28, 29, 30, 31, 32,
5234        );
5235        let r = _mm256_srli_si256::<3>(a);
5236        #[rustfmt::skip]
5237        let e = _mm256_setr_epi8(
5238            4, 5, 6, 7, 8, 9, 10, 11,
5239            12, 13, 14, 15, 16, 0, 0, 0,
5240            20, 21, 22, 23, 24, 25, 26, 27,
5241            28, 29, 30, 31, 32, 0, 0, 0,
5242        );
5243        assert_eq_m256i(r, e);
5244    }
5245
5246    #[simd_test(enable = "avx2")]
5247    fn test_mm256_srl_epi16() {
5248        let a = _mm256_set1_epi16(0xFF);
5249        let b = _mm_insert_epi16::<0>(_mm_set1_epi16(0), 4);
5250        let r = _mm256_srl_epi16(a, b);
5251        assert_eq_m256i(r, _mm256_set1_epi16(0xF));
5252    }
5253
5254    #[simd_test(enable = "avx2")]
5255    fn test_mm256_srl_epi32() {
5256        let a = _mm256_set1_epi32(0xFFFF);
5257        let b = _mm_insert_epi32::<0>(_mm_set1_epi32(0), 4);
5258        let r = _mm256_srl_epi32(a, b);
5259        assert_eq_m256i(r, _mm256_set1_epi32(0xFFF));
5260    }
5261
5262    #[simd_test(enable = "avx2")]
5263    fn test_mm256_srl_epi64() {
5264        let a = _mm256_set1_epi64x(0xFFFFFFFF);
5265        let b = _mm_setr_epi64x(4, 0);
5266        let r = _mm256_srl_epi64(a, b);
5267        assert_eq_m256i(r, _mm256_set1_epi64x(0xFFFFFFF));
5268    }
5269
5270    #[simd_test(enable = "avx2")]
5271    const fn test_mm256_srli_epi16() {
5272        assert_eq_m256i(
5273            _mm256_srli_epi16::<4>(_mm256_set1_epi16(0xFF)),
5274            _mm256_set1_epi16(0xF),
5275        );
5276    }
5277
5278    #[simd_test(enable = "avx2")]
5279    const fn test_mm256_srli_epi32() {
5280        assert_eq_m256i(
5281            _mm256_srli_epi32::<4>(_mm256_set1_epi32(0xFFFF)),
5282            _mm256_set1_epi32(0xFFF),
5283        );
5284    }
5285
5286    #[simd_test(enable = "avx2")]
5287    const fn test_mm256_srli_epi64() {
5288        assert_eq_m256i(
5289            _mm256_srli_epi64::<4>(_mm256_set1_epi64x(0xFFFFFFFF)),
5290            _mm256_set1_epi64x(0xFFFFFFF),
5291        );
5292    }
5293
5294    #[simd_test(enable = "avx2")]
5295    fn test_mm_srlv_epi32() {
5296        let a = _mm_set1_epi32(2);
5297        let count = _mm_set1_epi32(1);
5298        let r = _mm_srlv_epi32(a, count);
5299        let e = _mm_set1_epi32(1);
5300        assert_eq_m128i(r, e);
5301    }
5302
5303    #[simd_test(enable = "avx2")]
5304    fn test_mm256_srlv_epi32() {
5305        let a = _mm256_set1_epi32(2);
5306        let count = _mm256_set1_epi32(1);
5307        let r = _mm256_srlv_epi32(a, count);
5308        let e = _mm256_set1_epi32(1);
5309        assert_eq_m256i(r, e);
5310    }
5311
5312    #[simd_test(enable = "avx2")]
5313    fn test_mm_srlv_epi64() {
5314        let a = _mm_set_epi64x(4, 8);
5315        let b = _mm_set_epi64x(2, 1);
5316        let r = _mm_srlv_epi64(a, b);
5317        // Compare with the scalar version of the same computation.
5318        let e = _mm_set_epi64x(4i64.unbounded_shr(2), 8i64.unbounded_shr(1));
5319        assert_eq_m128i(r, e);
5320        // Compare with hardcoded output.
5321        let e = _mm_set_epi64x(1, 4);
5322        assert_eq_m128i(r, e);
5323
5324        // The shift has unbounded semantics: if the shift amount
5325        // is >= the number of bits the result is 0.
5326        let a = _mm_set_epi64x(i64::MAX, i64::MAX);
5327        let b = _mm_set_epi64x(64, 65);
5328        let r = _mm_sllv_epi64(a, b);
5329        let e = _mm_set_epi64x(i64::MAX.unbounded_shr(64), i64::MAX.unbounded_shr(65));
5330        assert_eq_m128i(r, e);
5331        let e = _mm_set_epi64x(0, 0);
5332        assert_eq_m128i(r, e);
5333    }
5334
5335    #[simd_test(enable = "avx2")]
5336    fn test_mm256_srlv_epi64() {
5337        let a = _mm256_set1_epi64x(2);
5338        let count = _mm256_set1_epi64x(1);
5339        let r = _mm256_srlv_epi64(a, count);
5340        let e = _mm256_set1_epi64x(1);
5341        assert_eq_m256i(r, e);
5342    }
5343
5344    #[simd_test(enable = "avx2")]
5345    fn test_mm256_stream_load_si256() {
5346        let a = _mm256_set_epi64x(5, 6, 7, 8);
5347        let r = unsafe { _mm256_stream_load_si256(core::ptr::addr_of!(a) as *const _) };
5348        assert_eq_m256i(a, r);
5349    }
5350
5351    #[simd_test(enable = "avx2")]
5352    const fn test_mm256_sub_epi16() {
5353        let a = _mm256_set1_epi16(4);
5354        let b = _mm256_set1_epi16(2);
5355        let r = _mm256_sub_epi16(a, b);
5356        assert_eq_m256i(r, b);
5357    }
5358
5359    #[simd_test(enable = "avx2")]
5360    const fn test_mm256_sub_epi32() {
5361        let a = _mm256_set1_epi32(4);
5362        let b = _mm256_set1_epi32(2);
5363        let r = _mm256_sub_epi32(a, b);
5364        assert_eq_m256i(r, b);
5365    }
5366
5367    #[simd_test(enable = "avx2")]
5368    const fn test_mm256_sub_epi64() {
5369        let a = _mm256_set1_epi64x(4);
5370        let b = _mm256_set1_epi64x(2);
5371        let r = _mm256_sub_epi64(a, b);
5372        assert_eq_m256i(r, b);
5373    }
5374
5375    #[simd_test(enable = "avx2")]
5376    const fn test_mm256_sub_epi8() {
5377        let a = _mm256_set1_epi8(4);
5378        let b = _mm256_set1_epi8(2);
5379        let r = _mm256_sub_epi8(a, b);
5380        assert_eq_m256i(r, b);
5381    }
5382
5383    #[simd_test(enable = "avx2")]
5384    const fn test_mm256_subs_epi16() {
5385        let a = _mm256_set1_epi16(4);
5386        let b = _mm256_set1_epi16(2);
5387        let r = _mm256_subs_epi16(a, b);
5388        assert_eq_m256i(r, b);
5389    }
5390
5391    #[simd_test(enable = "avx2")]
5392    const fn test_mm256_subs_epi8() {
5393        let a = _mm256_set1_epi8(4);
5394        let b = _mm256_set1_epi8(2);
5395        let r = _mm256_subs_epi8(a, b);
5396        assert_eq_m256i(r, b);
5397    }
5398
5399    #[simd_test(enable = "avx2")]
5400    const fn test_mm256_subs_epu16() {
5401        let a = _mm256_set1_epi16(4);
5402        let b = _mm256_set1_epi16(2);
5403        let r = _mm256_subs_epu16(a, b);
5404        assert_eq_m256i(r, b);
5405    }
5406
5407    #[simd_test(enable = "avx2")]
5408    const fn test_mm256_subs_epu8() {
5409        let a = _mm256_set1_epi8(4);
5410        let b = _mm256_set1_epi8(2);
5411        let r = _mm256_subs_epu8(a, b);
5412        assert_eq_m256i(r, b);
5413    }
5414
5415    #[simd_test(enable = "avx2")]
5416    const fn test_mm256_xor_si256() {
5417        let a = _mm256_set1_epi8(5);
5418        let b = _mm256_set1_epi8(3);
5419        let r = _mm256_xor_si256(a, b);
5420        assert_eq_m256i(r, _mm256_set1_epi8(6));
5421    }
5422
5423    #[simd_test(enable = "avx2")]
5424    const fn test_mm256_alignr_epi8() {
5425        #[rustfmt::skip]
5426        let a = _mm256_setr_epi8(
5427            1, 2, 3, 4, 5, 6, 7, 8,
5428            9, 10, 11, 12, 13, 14, 15, 16,
5429            17, 18, 19, 20, 21, 22, 23, 24,
5430            25, 26, 27, 28, 29, 30, 31, 32,
5431        );
5432        #[rustfmt::skip]
5433        let b = _mm256_setr_epi8(
5434            -1, -2, -3, -4, -5, -6, -7, -8,
5435            -9, -10, -11, -12, -13, -14, -15, -16,
5436            -17, -18, -19, -20, -21, -22, -23, -24,
5437            -25, -26, -27, -28, -29, -30, -31, -32,
5438        );
5439        let r = _mm256_alignr_epi8::<33>(a, b);
5440        assert_eq_m256i(r, _mm256_set1_epi8(0));
5441
5442        let r = _mm256_alignr_epi8::<17>(a, b);
5443        #[rustfmt::skip]
5444        let expected = _mm256_setr_epi8(
5445            2, 3, 4, 5, 6, 7, 8, 9,
5446            10, 11, 12, 13, 14, 15, 16, 0,
5447            18, 19, 20, 21, 22, 23, 24, 25,
5448            26, 27, 28, 29, 30, 31, 32, 0,
5449        );
5450        assert_eq_m256i(r, expected);
5451
5452        let r = _mm256_alignr_epi8::<4>(a, b);
5453        #[rustfmt::skip]
5454        let expected = _mm256_setr_epi8(
5455            -5, -6, -7, -8, -9, -10, -11, -12,
5456            -13, -14, -15, -16, 1, 2, 3, 4,
5457            -21, -22, -23, -24, -25, -26, -27, -28,
5458            -29, -30, -31, -32, 17, 18, 19, 20,
5459        );
5460        assert_eq_m256i(r, expected);
5461
5462        let r = _mm256_alignr_epi8::<15>(a, b);
5463        #[rustfmt::skip]
5464        let expected = _mm256_setr_epi8(
5465            -16, 1, 2, 3, 4, 5, 6, 7,
5466            8, 9, 10, 11, 12, 13, 14, 15,
5467            -32, 17, 18, 19, 20, 21, 22, 23,
5468            24, 25, 26, 27, 28, 29, 30, 31,
5469        );
5470        assert_eq_m256i(r, expected);
5471
5472        let r = _mm256_alignr_epi8::<0>(a, b);
5473        assert_eq_m256i(r, b);
5474
5475        let r = _mm256_alignr_epi8::<16>(a, b);
5476        assert_eq_m256i(r, a);
5477    }
5478
5479    #[simd_test(enable = "avx2")]
5480    fn test_mm256_shuffle_epi8() {
5481        #[rustfmt::skip]
5482        let a = _mm256_setr_epi8(
5483            1, 2, 3, 4, 5, 6, 7, 8,
5484            9, 10, 11, 12, 13, 14, 15, 16,
5485            17, 18, 19, 20, 21, 22, 23, 24,
5486            25, 26, 27, 28, 29, 30, 31, 32,
5487        );
5488        #[rustfmt::skip]
5489        let b = _mm256_setr_epi8(
5490            4, 128u8 as i8, 4, 3, 24, 12, 6, 19,
5491            12, 5, 5, 10, 4, 1, 8, 0,
5492            4, 128u8 as i8, 4, 3, 24, 12, 6, 19,
5493            12, 5, 5, 10, 4, 1, 8, 0,
5494        );
5495        #[rustfmt::skip]
5496        let expected = _mm256_setr_epi8(
5497            5, 0, 5, 4, 9, 13, 7, 4,
5498            13, 6, 6, 11, 5, 2, 9, 1,
5499            21, 0, 21, 20, 25, 29, 23, 20,
5500            29, 22, 22, 27, 21, 18, 25, 17,
5501        );
5502        let r = _mm256_shuffle_epi8(a, b);
5503        assert_eq_m256i(r, expected);
5504    }
5505
5506    #[simd_test(enable = "avx2")]
5507    fn test_mm256_permutevar8x32_epi32() {
5508        let a = _mm256_setr_epi32(100, 200, 300, 400, 500, 600, 700, 800);
5509        let b = _mm256_setr_epi32(5, 0, 5, 1, 7, 6, 3, 4);
5510        let expected = _mm256_setr_epi32(600, 100, 600, 200, 800, 700, 400, 500);
5511        let r = _mm256_permutevar8x32_epi32(a, b);
5512        assert_eq_m256i(r, expected);
5513    }
5514
5515    #[simd_test(enable = "avx2")]
5516    const fn test_mm256_permute4x64_epi64() {
5517        let a = _mm256_setr_epi64x(100, 200, 300, 400);
5518        let expected = _mm256_setr_epi64x(400, 100, 200, 100);
5519        let r = _mm256_permute4x64_epi64::<0b00010011>(a);
5520        assert_eq_m256i(r, expected);
5521    }
5522
5523    #[simd_test(enable = "avx2")]
5524    const fn test_mm256_permute2x128_si256() {
5525        let a = _mm256_setr_epi64x(100, 200, 500, 600);
5526        let b = _mm256_setr_epi64x(300, 400, 700, 800);
5527        let r = _mm256_permute2x128_si256::<0b00_01_00_11>(a, b);
5528        let e = _mm256_setr_epi64x(700, 800, 500, 600);
5529        assert_eq_m256i(r, e);
5530    }
5531
5532    #[simd_test(enable = "avx2")]
5533    const fn test_mm256_permute4x64_pd() {
5534        let a = _mm256_setr_pd(1., 2., 3., 4.);
5535        let r = _mm256_permute4x64_pd::<0b00_01_00_11>(a);
5536        let e = _mm256_setr_pd(4., 1., 2., 1.);
5537        assert_eq_m256d(r, e);
5538    }
5539
5540    #[simd_test(enable = "avx2")]
5541    fn test_mm256_permutevar8x32_ps() {
5542        let a = _mm256_setr_ps(1., 2., 3., 4., 5., 6., 7., 8.);
5543        let b = _mm256_setr_epi32(5, 0, 5, 1, 7, 6, 3, 4);
5544        let r = _mm256_permutevar8x32_ps(a, b);
5545        let e = _mm256_setr_ps(6., 1., 6., 2., 8., 7., 4., 5.);
5546        assert_eq_m256(r, e);
5547    }
5548
5549    #[simd_test(enable = "avx2")]
5550    fn test_mm_i32gather_epi32() {
5551        let arr: [i32; 128] = core::array::from_fn(|i| i as i32);
5552        // A multiplier of 4 is word-addressing
5553        let r = unsafe { _mm_i32gather_epi32::<4>(arr.as_ptr(), _mm_setr_epi32(0, 16, 32, 48)) };
5554        assert_eq_m128i(r, _mm_setr_epi32(0, 16, 32, 48));
5555    }
5556
5557    #[simd_test(enable = "avx2")]
5558    fn test_mm_mask_i32gather_epi32() {
5559        let arr: [i32; 128] = core::array::from_fn(|i| i as i32);
5560        // A multiplier of 4 is word-addressing
5561        let r = unsafe {
5562            _mm_mask_i32gather_epi32::<4>(
5563                _mm_set1_epi32(256),
5564                arr.as_ptr(),
5565                _mm_setr_epi32(0, 16, 64, 96),
5566                _mm_setr_epi32(-1, -1, -1, 0),
5567            )
5568        };
5569        assert_eq_m128i(r, _mm_setr_epi32(0, 16, 64, 256));
5570    }
5571
5572    #[simd_test(enable = "avx2")]
5573    fn test_mm256_i32gather_epi32() {
5574        let arr: [i32; 128] = core::array::from_fn(|i| i as i32);
5575        // A multiplier of 4 is word-addressing
5576        let r = unsafe {
5577            _mm256_i32gather_epi32::<4>(arr.as_ptr(), _mm256_setr_epi32(0, 16, 32, 48, 1, 2, 3, 4))
5578        };
5579        assert_eq_m256i(r, _mm256_setr_epi32(0, 16, 32, 48, 1, 2, 3, 4));
5580    }
5581
5582    #[simd_test(enable = "avx2")]
5583    fn test_mm256_mask_i32gather_epi32() {
5584        let arr: [i32; 128] = core::array::from_fn(|i| i as i32);
5585        // A multiplier of 4 is word-addressing
5586        let r = unsafe {
5587            _mm256_mask_i32gather_epi32::<4>(
5588                _mm256_set1_epi32(256),
5589                arr.as_ptr(),
5590                _mm256_setr_epi32(0, 16, 64, 96, 0, 0, 0, 0),
5591                _mm256_setr_epi32(-1, -1, -1, 0, 0, 0, 0, 0),
5592            )
5593        };
5594        assert_eq_m256i(r, _mm256_setr_epi32(0, 16, 64, 256, 256, 256, 256, 256));
5595    }
5596
5597    #[simd_test(enable = "avx2")]
5598    fn test_mm_i32gather_ps() {
5599        let arr: [f32; 128] = core::array::from_fn(|i| i as f32);
5600        // A multiplier of 4 is word-addressing for f32s
5601        let r = unsafe { _mm_i32gather_ps::<4>(arr.as_ptr(), _mm_setr_epi32(0, 16, 32, 48)) };
5602        assert_eq_m128(r, _mm_setr_ps(0.0, 16.0, 32.0, 48.0));
5603    }
5604
5605    #[simd_test(enable = "avx2")]
5606    fn test_mm_mask_i32gather_ps() {
5607        let arr: [f32; 128] = core::array::from_fn(|i| i as f32);
5608        // A multiplier of 4 is word-addressing for f32s
5609        let r = unsafe {
5610            _mm_mask_i32gather_ps::<4>(
5611                _mm_set1_ps(256.0),
5612                arr.as_ptr(),
5613                _mm_setr_epi32(0, 16, 64, 96),
5614                _mm_setr_ps(-1.0, -1.0, -1.0, 0.0),
5615            )
5616        };
5617        assert_eq_m128(r, _mm_setr_ps(0.0, 16.0, 64.0, 256.0));
5618    }
5619
5620    #[simd_test(enable = "avx2")]
5621    fn test_mm256_i32gather_ps() {
5622        let arr: [f32; 128] = core::array::from_fn(|i| i as f32);
5623        // A multiplier of 4 is word-addressing for f32s
5624        let r = unsafe {
5625            _mm256_i32gather_ps::<4>(arr.as_ptr(), _mm256_setr_epi32(0, 16, 32, 48, 1, 2, 3, 4))
5626        };
5627        assert_eq_m256(r, _mm256_setr_ps(0.0, 16.0, 32.0, 48.0, 1.0, 2.0, 3.0, 4.0));
5628    }
5629
5630    #[simd_test(enable = "avx2")]
5631    fn test_mm256_mask_i32gather_ps() {
5632        let arr: [f32; 128] = core::array::from_fn(|i| i as f32);
5633        // A multiplier of 4 is word-addressing for f32s
5634        let r = unsafe {
5635            _mm256_mask_i32gather_ps::<4>(
5636                _mm256_set1_ps(256.0),
5637                arr.as_ptr(),
5638                _mm256_setr_epi32(0, 16, 64, 96, 0, 0, 0, 0),
5639                _mm256_setr_ps(-1.0, -1.0, -1.0, 0.0, 0.0, 0.0, 0.0, 0.0),
5640            )
5641        };
5642        assert_eq_m256(
5643            r,
5644            _mm256_setr_ps(0.0, 16.0, 64.0, 256.0, 256.0, 256.0, 256.0, 256.0),
5645        );
5646    }
5647
5648    #[simd_test(enable = "avx2")]
5649    fn test_mm_i32gather_epi64() {
5650        let arr: [i64; 128] = core::array::from_fn(|i| i as i64);
5651        // A multiplier of 8 is word-addressing for i64s
5652        let r = unsafe { _mm_i32gather_epi64::<8>(arr.as_ptr(), _mm_setr_epi32(0, 16, 0, 0)) };
5653        assert_eq_m128i(r, _mm_setr_epi64x(0, 16));
5654    }
5655
5656    #[simd_test(enable = "avx2")]
5657    fn test_mm_mask_i32gather_epi64() {
5658        let arr: [i64; 128] = core::array::from_fn(|i| i as i64);
5659        // A multiplier of 8 is word-addressing for i64s
5660        let r = unsafe {
5661            _mm_mask_i32gather_epi64::<8>(
5662                _mm_set1_epi64x(256),
5663                arr.as_ptr(),
5664                _mm_setr_epi32(16, 16, 16, 16),
5665                _mm_setr_epi64x(-1, 0),
5666            )
5667        };
5668        assert_eq_m128i(r, _mm_setr_epi64x(16, 256));
5669    }
5670
5671    #[simd_test(enable = "avx2")]
5672    fn test_mm256_i32gather_epi64() {
5673        let arr: [i64; 128] = core::array::from_fn(|i| i as i64);
5674        // A multiplier of 8 is word-addressing for i64s
5675        let r = unsafe { _mm256_i32gather_epi64::<8>(arr.as_ptr(), _mm_setr_epi32(0, 16, 32, 48)) };
5676        assert_eq_m256i(r, _mm256_setr_epi64x(0, 16, 32, 48));
5677    }
5678
5679    #[simd_test(enable = "avx2")]
5680    fn test_mm256_mask_i32gather_epi64() {
5681        let arr: [i64; 128] = core::array::from_fn(|i| i as i64);
5682        // A multiplier of 8 is word-addressing for i64s
5683        let r = unsafe {
5684            _mm256_mask_i32gather_epi64::<8>(
5685                _mm256_set1_epi64x(256),
5686                arr.as_ptr(),
5687                _mm_setr_epi32(0, 16, 64, 96),
5688                _mm256_setr_epi64x(-1, -1, -1, 0),
5689            )
5690        };
5691        assert_eq_m256i(r, _mm256_setr_epi64x(0, 16, 64, 256));
5692    }
5693
5694    #[simd_test(enable = "avx2")]
5695    fn test_mm_i32gather_pd() {
5696        let arr: [f64; 128] = core::array::from_fn(|i| i as f64);
5697        // A multiplier of 8 is word-addressing for f64s
5698        let r = unsafe { _mm_i32gather_pd::<8>(arr.as_ptr(), _mm_setr_epi32(0, 16, 0, 0)) };
5699        assert_eq_m128d(r, _mm_setr_pd(0.0, 16.0));
5700    }
5701
5702    #[simd_test(enable = "avx2")]
5703    fn test_mm_mask_i32gather_pd() {
5704        let arr: [f64; 128] = core::array::from_fn(|i| i as f64);
5705        // A multiplier of 8 is word-addressing for f64s
5706        let r = unsafe {
5707            _mm_mask_i32gather_pd::<8>(
5708                _mm_set1_pd(256.0),
5709                arr.as_ptr(),
5710                _mm_setr_epi32(16, 16, 16, 16),
5711                _mm_setr_pd(-1.0, 0.0),
5712            )
5713        };
5714        assert_eq_m128d(r, _mm_setr_pd(16.0, 256.0));
5715    }
5716
5717    #[simd_test(enable = "avx2")]
5718    fn test_mm256_i32gather_pd() {
5719        let arr: [f64; 128] = core::array::from_fn(|i| i as f64);
5720        // A multiplier of 8 is word-addressing for f64s
5721        let r = unsafe { _mm256_i32gather_pd::<8>(arr.as_ptr(), _mm_setr_epi32(0, 16, 32, 48)) };
5722        assert_eq_m256d(r, _mm256_setr_pd(0.0, 16.0, 32.0, 48.0));
5723    }
5724
5725    #[simd_test(enable = "avx2")]
5726    fn test_mm256_mask_i32gather_pd() {
5727        let arr: [f64; 128] = core::array::from_fn(|i| i as f64);
5728        // A multiplier of 8 is word-addressing for f64s
5729        let r = unsafe {
5730            _mm256_mask_i32gather_pd::<8>(
5731                _mm256_set1_pd(256.0),
5732                arr.as_ptr(),
5733                _mm_setr_epi32(0, 16, 64, 96),
5734                _mm256_setr_pd(-1.0, -1.0, -1.0, 0.0),
5735            )
5736        };
5737        assert_eq_m256d(r, _mm256_setr_pd(0.0, 16.0, 64.0, 256.0));
5738    }
5739
5740    #[simd_test(enable = "avx2")]
5741    fn test_mm_i64gather_epi32() {
5742        let arr: [i32; 128] = core::array::from_fn(|i| i as i32);
5743        // A multiplier of 4 is word-addressing
5744        let r = unsafe { _mm_i64gather_epi32::<4>(arr.as_ptr(), _mm_setr_epi64x(0, 16)) };
5745        assert_eq_m128i(r, _mm_setr_epi32(0, 16, 0, 0));
5746    }
5747
5748    #[simd_test(enable = "avx2")]
5749    fn test_mm_mask_i64gather_epi32() {
5750        let arr: [i32; 128] = core::array::from_fn(|i| i as i32);
5751        // A multiplier of 4 is word-addressing
5752        let r = unsafe {
5753            _mm_mask_i64gather_epi32::<4>(
5754                _mm_set1_epi32(256),
5755                arr.as_ptr(),
5756                _mm_setr_epi64x(0, 16),
5757                _mm_setr_epi32(-1, 0, -1, 0),
5758            )
5759        };
5760        assert_eq_m128i(r, _mm_setr_epi32(0, 256, 0, 0));
5761    }
5762
5763    #[simd_test(enable = "avx2")]
5764    fn test_mm256_i64gather_epi32() {
5765        let arr: [i32; 128] = core::array::from_fn(|i| i as i32);
5766        // A multiplier of 4 is word-addressing
5767        let r =
5768            unsafe { _mm256_i64gather_epi32::<4>(arr.as_ptr(), _mm256_setr_epi64x(0, 16, 32, 48)) };
5769        assert_eq_m128i(r, _mm_setr_epi32(0, 16, 32, 48));
5770    }
5771
5772    #[simd_test(enable = "avx2")]
5773    fn test_mm256_mask_i64gather_epi32() {
5774        let arr: [i32; 128] = core::array::from_fn(|i| i as i32);
5775        // A multiplier of 4 is word-addressing
5776        let r = unsafe {
5777            _mm256_mask_i64gather_epi32::<4>(
5778                _mm_set1_epi32(256),
5779                arr.as_ptr(),
5780                _mm256_setr_epi64x(0, 16, 64, 96),
5781                _mm_setr_epi32(-1, -1, -1, 0),
5782            )
5783        };
5784        assert_eq_m128i(r, _mm_setr_epi32(0, 16, 64, 256));
5785    }
5786
5787    #[simd_test(enable = "avx2")]
5788    fn test_mm_i64gather_ps() {
5789        let arr: [f32; 128] = core::array::from_fn(|i| i as f32);
5790        // A multiplier of 4 is word-addressing for f32s
5791        let r = unsafe { _mm_i64gather_ps::<4>(arr.as_ptr(), _mm_setr_epi64x(0, 16)) };
5792        assert_eq_m128(r, _mm_setr_ps(0.0, 16.0, 0.0, 0.0));
5793    }
5794
5795    #[simd_test(enable = "avx2")]
5796    fn test_mm_mask_i64gather_ps() {
5797        let arr: [f32; 128] = core::array::from_fn(|i| i as f32);
5798        // A multiplier of 4 is word-addressing for f32s
5799        let r = unsafe {
5800            _mm_mask_i64gather_ps::<4>(
5801                _mm_set1_ps(256.0),
5802                arr.as_ptr(),
5803                _mm_setr_epi64x(0, 16),
5804                _mm_setr_ps(-1.0, 0.0, -1.0, 0.0),
5805            )
5806        };
5807        assert_eq_m128(r, _mm_setr_ps(0.0, 256.0, 0.0, 0.0));
5808    }
5809
5810    #[simd_test(enable = "avx2")]
5811    fn test_mm256_i64gather_ps() {
5812        let arr: [f32; 128] = core::array::from_fn(|i| i as f32);
5813        // A multiplier of 4 is word-addressing for f32s
5814        let r =
5815            unsafe { _mm256_i64gather_ps::<4>(arr.as_ptr(), _mm256_setr_epi64x(0, 16, 32, 48)) };
5816        assert_eq_m128(r, _mm_setr_ps(0.0, 16.0, 32.0, 48.0));
5817    }
5818
5819    #[simd_test(enable = "avx2")]
5820    fn test_mm256_mask_i64gather_ps() {
5821        let arr: [f32; 128] = core::array::from_fn(|i| i as f32);
5822        // A multiplier of 4 is word-addressing for f32s
5823        let r = unsafe {
5824            _mm256_mask_i64gather_ps::<4>(
5825                _mm_set1_ps(256.0),
5826                arr.as_ptr(),
5827                _mm256_setr_epi64x(0, 16, 64, 96),
5828                _mm_setr_ps(-1.0, -1.0, -1.0, 0.0),
5829            )
5830        };
5831        assert_eq_m128(r, _mm_setr_ps(0.0, 16.0, 64.0, 256.0));
5832    }
5833
5834    #[simd_test(enable = "avx2")]
5835    fn test_mm_i64gather_epi64() {
5836        let arr: [i64; 128] = core::array::from_fn(|i| i as i64);
5837        // A multiplier of 8 is word-addressing for i64s
5838        let r = unsafe { _mm_i64gather_epi64::<8>(arr.as_ptr(), _mm_setr_epi64x(0, 16)) };
5839        assert_eq_m128i(r, _mm_setr_epi64x(0, 16));
5840    }
5841
5842    #[simd_test(enable = "avx2")]
5843    fn test_mm_mask_i64gather_epi64() {
5844        let arr: [i64; 128] = core::array::from_fn(|i| i as i64);
5845        // A multiplier of 8 is word-addressing for i64s
5846        let r = unsafe {
5847            _mm_mask_i64gather_epi64::<8>(
5848                _mm_set1_epi64x(256),
5849                arr.as_ptr(),
5850                _mm_setr_epi64x(16, 16),
5851                _mm_setr_epi64x(-1, 0),
5852            )
5853        };
5854        assert_eq_m128i(r, _mm_setr_epi64x(16, 256));
5855    }
5856
5857    #[simd_test(enable = "avx2")]
5858    fn test_mm256_i64gather_epi64() {
5859        let arr: [i64; 128] = core::array::from_fn(|i| i as i64);
5860        // A multiplier of 8 is word-addressing for i64s
5861        let r =
5862            unsafe { _mm256_i64gather_epi64::<8>(arr.as_ptr(), _mm256_setr_epi64x(0, 16, 32, 48)) };
5863        assert_eq_m256i(r, _mm256_setr_epi64x(0, 16, 32, 48));
5864    }
5865
5866    #[simd_test(enable = "avx2")]
5867    fn test_mm256_mask_i64gather_epi64() {
5868        let arr: [i64; 128] = core::array::from_fn(|i| i as i64);
5869        // A multiplier of 8 is word-addressing for i64s
5870        let r = unsafe {
5871            _mm256_mask_i64gather_epi64::<8>(
5872                _mm256_set1_epi64x(256),
5873                arr.as_ptr(),
5874                _mm256_setr_epi64x(0, 16, 64, 96),
5875                _mm256_setr_epi64x(-1, -1, -1, 0),
5876            )
5877        };
5878        assert_eq_m256i(r, _mm256_setr_epi64x(0, 16, 64, 256));
5879    }
5880
5881    #[simd_test(enable = "avx2")]
5882    fn test_mm_i64gather_pd() {
5883        let arr: [f64; 128] = core::array::from_fn(|i| i as f64);
5884        // A multiplier of 8 is word-addressing for f64s
5885        let r = unsafe { _mm_i64gather_pd::<8>(arr.as_ptr(), _mm_setr_epi64x(0, 16)) };
5886        assert_eq_m128d(r, _mm_setr_pd(0.0, 16.0));
5887    }
5888
5889    #[simd_test(enable = "avx2")]
5890    fn test_mm_mask_i64gather_pd() {
5891        let arr: [f64; 128] = core::array::from_fn(|i| i as f64);
5892        // A multiplier of 8 is word-addressing for f64s
5893        let r = unsafe {
5894            _mm_mask_i64gather_pd::<8>(
5895                _mm_set1_pd(256.0),
5896                arr.as_ptr(),
5897                _mm_setr_epi64x(16, 16),
5898                _mm_setr_pd(-1.0, 0.0),
5899            )
5900        };
5901        assert_eq_m128d(r, _mm_setr_pd(16.0, 256.0));
5902    }
5903
5904    #[simd_test(enable = "avx2")]
5905    fn test_mm256_i64gather_pd() {
5906        let arr: [f64; 128] = core::array::from_fn(|i| i as f64);
5907        // A multiplier of 8 is word-addressing for f64s
5908        let r =
5909            unsafe { _mm256_i64gather_pd::<8>(arr.as_ptr(), _mm256_setr_epi64x(0, 16, 32, 48)) };
5910        assert_eq_m256d(r, _mm256_setr_pd(0.0, 16.0, 32.0, 48.0));
5911    }
5912
5913    #[simd_test(enable = "avx2")]
5914    fn test_mm256_mask_i64gather_pd() {
5915        let arr: [f64; 128] = core::array::from_fn(|i| i as f64);
5916        // A multiplier of 8 is word-addressing for f64s
5917        let r = unsafe {
5918            _mm256_mask_i64gather_pd::<8>(
5919                _mm256_set1_pd(256.0),
5920                arr.as_ptr(),
5921                _mm256_setr_epi64x(0, 16, 64, 96),
5922                _mm256_setr_pd(-1.0, -1.0, -1.0, 0.0),
5923            )
5924        };
5925        assert_eq_m256d(r, _mm256_setr_pd(0.0, 16.0, 64.0, 256.0));
5926    }
5927
5928    #[simd_test(enable = "avx2")]
5929    const fn test_mm256_extract_epi8() {
5930        #[rustfmt::skip]
5931        let a = _mm256_setr_epi8(
5932            -1, 1, 2, 3, 4, 5, 6, 7,
5933            8, 9, 10, 11, 12, 13, 14, 15,
5934            16, 17, 18, 19, 20, 21, 22, 23,
5935            24, 25, 26, 27, 28, 29, 30, 31
5936        );
5937        let r1 = _mm256_extract_epi8::<0>(a);
5938        let r2 = _mm256_extract_epi8::<3>(a);
5939        assert_eq!(r1, 0xFF);
5940        assert_eq!(r2, 3);
5941    }
5942
5943    #[simd_test(enable = "avx2")]
5944    const fn test_mm256_extract_epi16() {
5945        #[rustfmt::skip]
5946        let a = _mm256_setr_epi16(
5947            -1, 1, 2, 3, 4, 5, 6, 7,
5948            8, 9, 10, 11, 12, 13, 14, 15,
5949        );
5950        let r1 = _mm256_extract_epi16::<0>(a);
5951        let r2 = _mm256_extract_epi16::<3>(a);
5952        assert_eq!(r1, 0xFFFF);
5953        assert_eq!(r2, 3);
5954    }
5955}