Eigen  3.2.93
AVX/Complex.h
1 // This file is part of Eigen, a lightweight C++ template library
2 // for linear algebra.
3 //
4 // Copyright (C) 2014 Benoit Steiner (benoit.steiner.goog@gmail.com)
5 //
6 // This Source Code Form is subject to the terms of the Mozilla
7 // Public License v. 2.0. If a copy of the MPL was not distributed
8 // with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
9 
10 #ifndef EIGEN_COMPLEX_AVX_H
11 #define EIGEN_COMPLEX_AVX_H
12 
13 namespace Eigen {
14 
15 namespace internal {
16 
17 //---------- float ----------
18 struct Packet4cf
19 {
20  EIGEN_STRONG_INLINE Packet4cf() {}
21  EIGEN_STRONG_INLINE explicit Packet4cf(const __m256& a) : v(a) {}
22  __m256 v;
23 };
24 
25 template<> struct packet_traits<std::complex<float> > : default_packet_traits
26 {
27  typedef Packet4cf type;
28  typedef Packet2cf half;
29  enum {
30  Vectorizable = 1,
31  AlignedOnScalar = 1,
32  size = 4,
33  HasHalfPacket = 1,
34 
35  HasAdd = 1,
36  HasSub = 1,
37  HasMul = 1,
38  HasDiv = 1,
39  HasNegate = 1,
40  HasAbs = 0,
41  HasAbs2 = 0,
42  HasMin = 0,
43  HasMax = 0,
44  HasSetLinear = 0
45  };
46 };
47 
48 template<> struct unpacket_traits<Packet4cf> { typedef std::complex<float> type; enum {size=4, alignment=Aligned32}; typedef Packet2cf half; };
49 
50 template<> EIGEN_STRONG_INLINE Packet4cf padd<Packet4cf>(const Packet4cf& a, const Packet4cf& b) { return Packet4cf(_mm256_add_ps(a.v,b.v)); }
51 template<> EIGEN_STRONG_INLINE Packet4cf psub<Packet4cf>(const Packet4cf& a, const Packet4cf& b) { return Packet4cf(_mm256_sub_ps(a.v,b.v)); }
52 template<> EIGEN_STRONG_INLINE Packet4cf pnegate(const Packet4cf& a)
53 {
54  return Packet4cf(pnegate(a.v));
55 }
56 template<> EIGEN_STRONG_INLINE Packet4cf pconj(const Packet4cf& a)
57 {
58  const __m256 mask = _mm256_castsi256_ps(_mm256_setr_epi32(0x00000000,0x80000000,0x00000000,0x80000000,0x00000000,0x80000000,0x00000000,0x80000000));
59  return Packet4cf(_mm256_xor_ps(a.v,mask));
60 }
61 
62 template<> EIGEN_STRONG_INLINE Packet4cf pmul<Packet4cf>(const Packet4cf& a, const Packet4cf& b)
63 {
64  __m256 tmp1 = _mm256_mul_ps(_mm256_moveldup_ps(a.v), b.v);
65  __m256 tmp2 = _mm256_mul_ps(_mm256_movehdup_ps(a.v), _mm256_permute_ps(b.v, _MM_SHUFFLE(2,3,0,1)));
66  __m256 result = _mm256_addsub_ps(tmp1, tmp2);
67  return Packet4cf(result);
68 }
69 
70 template<> EIGEN_STRONG_INLINE Packet4cf pand <Packet4cf>(const Packet4cf& a, const Packet4cf& b) { return Packet4cf(_mm256_and_ps(a.v,b.v)); }
71 template<> EIGEN_STRONG_INLINE Packet4cf por <Packet4cf>(const Packet4cf& a, const Packet4cf& b) { return Packet4cf(_mm256_or_ps(a.v,b.v)); }
72 template<> EIGEN_STRONG_INLINE Packet4cf pxor <Packet4cf>(const Packet4cf& a, const Packet4cf& b) { return Packet4cf(_mm256_xor_ps(a.v,b.v)); }
73 template<> EIGEN_STRONG_INLINE Packet4cf pandnot<Packet4cf>(const Packet4cf& a, const Packet4cf& b) { return Packet4cf(_mm256_andnot_ps(a.v,b.v)); }
74 
75 template<> EIGEN_STRONG_INLINE Packet4cf pload <Packet4cf>(const std::complex<float>* from) { EIGEN_DEBUG_ALIGNED_LOAD return Packet4cf(pload<Packet8f>(&numext::real_ref(*from))); }
76 template<> EIGEN_STRONG_INLINE Packet4cf ploadu<Packet4cf>(const std::complex<float>* from) { EIGEN_DEBUG_UNALIGNED_LOAD return Packet4cf(ploadu<Packet8f>(&numext::real_ref(*from))); }
77 
78 
79 template<> EIGEN_STRONG_INLINE Packet4cf pset1<Packet4cf>(const std::complex<float>& from)
80 {
81  return Packet4cf(_mm256_castpd_ps(_mm256_broadcast_sd((const double*)(const void*)&from)));
82 }
83 
84 template<> EIGEN_STRONG_INLINE Packet4cf ploaddup<Packet4cf>(const std::complex<float>* from)
85 {
86  // FIXME The following might be optimized using _mm256_movedup_pd
87  Packet2cf a = ploaddup<Packet2cf>(from);
88  Packet2cf b = ploaddup<Packet2cf>(from+1);
89  return Packet4cf(_mm256_insertf128_ps(_mm256_castps128_ps256(a.v), b.v, 1));
90 }
91 
92 template<> EIGEN_STRONG_INLINE void pstore <std::complex<float> >(std::complex<float>* to, const Packet4cf& from) { EIGEN_DEBUG_ALIGNED_STORE pstore(&numext::real_ref(*to), from.v); }
93 template<> EIGEN_STRONG_INLINE void pstoreu<std::complex<float> >(std::complex<float>* to, const Packet4cf& from) { EIGEN_DEBUG_UNALIGNED_STORE pstoreu(&numext::real_ref(*to), from.v); }
94 
95 template<> EIGEN_DEVICE_FUNC inline Packet4cf pgather<std::complex<float>, Packet4cf>(const std::complex<float>* from, Index stride)
96 {
97  return Packet4cf(_mm256_set_ps(std::imag(from[3*stride]), std::real(from[3*stride]),
98  std::imag(from[2*stride]), std::real(from[2*stride]),
99  std::imag(from[1*stride]), std::real(from[1*stride]),
100  std::imag(from[0*stride]), std::real(from[0*stride])));
101 }
102 
103 template<> EIGEN_DEVICE_FUNC inline void pscatter<std::complex<float>, Packet4cf>(std::complex<float>* to, const Packet4cf& from, Index stride)
104 {
105  __m128 low = _mm256_extractf128_ps(from.v, 0);
106  to[stride*0] = std::complex<float>(_mm_cvtss_f32(_mm_shuffle_ps(low, low, 0)),
107  _mm_cvtss_f32(_mm_shuffle_ps(low, low, 1)));
108  to[stride*1] = std::complex<float>(_mm_cvtss_f32(_mm_shuffle_ps(low, low, 2)),
109  _mm_cvtss_f32(_mm_shuffle_ps(low, low, 3)));
110 
111  __m128 high = _mm256_extractf128_ps(from.v, 1);
112  to[stride*2] = std::complex<float>(_mm_cvtss_f32(_mm_shuffle_ps(high, high, 0)),
113  _mm_cvtss_f32(_mm_shuffle_ps(high, high, 1)));
114  to[stride*3] = std::complex<float>(_mm_cvtss_f32(_mm_shuffle_ps(high, high, 2)),
115  _mm_cvtss_f32(_mm_shuffle_ps(high, high, 3)));
116 
117 }
118 
119 template<> EIGEN_STRONG_INLINE std::complex<float> pfirst<Packet4cf>(const Packet4cf& a)
120 {
121  return pfirst(Packet2cf(_mm256_castps256_ps128(a.v)));
122 }
123 
124 template<> EIGEN_STRONG_INLINE Packet4cf preverse(const Packet4cf& a) {
125  __m128 low = _mm256_extractf128_ps(a.v, 0);
126  __m128 high = _mm256_extractf128_ps(a.v, 1);
127  __m128d lowd = _mm_castps_pd(low);
128  __m128d highd = _mm_castps_pd(high);
129  low = _mm_castpd_ps(_mm_shuffle_pd(lowd,lowd,0x1));
130  high = _mm_castpd_ps(_mm_shuffle_pd(highd,highd,0x1));
131  __m256 result = _mm256_setzero_ps();
132  result = _mm256_insertf128_ps(result, low, 1);
133  result = _mm256_insertf128_ps(result, high, 0);
134  return Packet4cf(result);
135 }
136 
137 template<> EIGEN_STRONG_INLINE std::complex<float> predux<Packet4cf>(const Packet4cf& a)
138 {
139  return predux(padd(Packet2cf(_mm256_extractf128_ps(a.v,0)),
140  Packet2cf(_mm256_extractf128_ps(a.v,1))));
141 }
142 
143 template<> EIGEN_STRONG_INLINE Packet4cf preduxp<Packet4cf>(const Packet4cf* vecs)
144 {
145  Packet8f t0 = _mm256_shuffle_ps(vecs[0].v, vecs[0].v, _MM_SHUFFLE(3, 1, 2 ,0));
146  Packet8f t1 = _mm256_shuffle_ps(vecs[1].v, vecs[1].v, _MM_SHUFFLE(3, 1, 2 ,0));
147  t0 = _mm256_hadd_ps(t0,t1);
148  Packet8f t2 = _mm256_shuffle_ps(vecs[2].v, vecs[2].v, _MM_SHUFFLE(3, 1, 2 ,0));
149  Packet8f t3 = _mm256_shuffle_ps(vecs[3].v, vecs[3].v, _MM_SHUFFLE(3, 1, 2 ,0));
150  t2 = _mm256_hadd_ps(t2,t3);
151 
152  t1 = _mm256_permute2f128_ps(t0,t2, 0 + (2<<4));
153  t3 = _mm256_permute2f128_ps(t0,t2, 1 + (3<<4));
154 
155  return Packet4cf(_mm256_add_ps(t1,t3));
156 }
157 
158 template<> EIGEN_STRONG_INLINE std::complex<float> predux_mul<Packet4cf>(const Packet4cf& a)
159 {
160  return predux_mul(pmul(Packet2cf(_mm256_extractf128_ps(a.v, 0)),
161  Packet2cf(_mm256_extractf128_ps(a.v, 1))));
162 }
163 
164 template<int Offset>
165 struct palign_impl<Offset,Packet4cf>
166 {
167  static EIGEN_STRONG_INLINE void run(Packet4cf& first, const Packet4cf& second)
168  {
169  if (Offset==0) return;
170  palign_impl<Offset*2,Packet8f>::run(first.v, second.v);
171  }
172 };
173 
174 template<> struct conj_helper<Packet4cf, Packet4cf, false,true>
175 {
176  EIGEN_STRONG_INLINE Packet4cf pmadd(const Packet4cf& x, const Packet4cf& y, const Packet4cf& c) const
177  { return padd(pmul(x,y),c); }
178 
179  EIGEN_STRONG_INLINE Packet4cf pmul(const Packet4cf& a, const Packet4cf& b) const
180  {
181  return internal::pmul(a, pconj(b));
182  }
183 };
184 
185 template<> struct conj_helper<Packet4cf, Packet4cf, true,false>
186 {
187  EIGEN_STRONG_INLINE Packet4cf pmadd(const Packet4cf& x, const Packet4cf& y, const Packet4cf& c) const
188  { return padd(pmul(x,y),c); }
189 
190  EIGEN_STRONG_INLINE Packet4cf pmul(const Packet4cf& a, const Packet4cf& b) const
191  {
192  return internal::pmul(pconj(a), b);
193  }
194 };
195 
196 template<> struct conj_helper<Packet4cf, Packet4cf, true,true>
197 {
198  EIGEN_STRONG_INLINE Packet4cf pmadd(const Packet4cf& x, const Packet4cf& y, const Packet4cf& c) const
199  { return padd(pmul(x,y),c); }
200 
201  EIGEN_STRONG_INLINE Packet4cf pmul(const Packet4cf& a, const Packet4cf& b) const
202  {
203  return pconj(internal::pmul(a, b));
204  }
205 };
206 
207 template<> struct conj_helper<Packet8f, Packet4cf, false,false>
208 {
209  EIGEN_STRONG_INLINE Packet4cf pmadd(const Packet8f& x, const Packet4cf& y, const Packet4cf& c) const
210  { return padd(c, pmul(x,y)); }
211 
212  EIGEN_STRONG_INLINE Packet4cf pmul(const Packet8f& x, const Packet4cf& y) const
213  { return Packet4cf(Eigen::internal::pmul(x, y.v)); }
214 };
215 
216 template<> struct conj_helper<Packet4cf, Packet8f, false,false>
217 {
218  EIGEN_STRONG_INLINE Packet4cf pmadd(const Packet4cf& x, const Packet8f& y, const Packet4cf& c) const
219  { return padd(c, pmul(x,y)); }
220 
221  EIGEN_STRONG_INLINE Packet4cf pmul(const Packet4cf& x, const Packet8f& y) const
222  { return Packet4cf(Eigen::internal::pmul(x.v, y)); }
223 };
224 
225 template<> EIGEN_STRONG_INLINE Packet4cf pdiv<Packet4cf>(const Packet4cf& a, const Packet4cf& b)
226 {
227  Packet4cf num = pmul(a, pconj(b));
228  __m256 tmp = _mm256_mul_ps(b.v, b.v);
229  __m256 tmp2 = _mm256_shuffle_ps(tmp,tmp,0xB1);
230  __m256 denom = _mm256_add_ps(tmp, tmp2);
231  return Packet4cf(_mm256_div_ps(num.v, denom));
232 }
233 
234 template<> EIGEN_STRONG_INLINE Packet4cf pcplxflip<Packet4cf>(const Packet4cf& x)
235 {
236  return Packet4cf(_mm256_shuffle_ps(x.v, x.v, _MM_SHUFFLE(2, 3, 0 ,1)));
237 }
238 
239 //---------- double ----------
240 struct Packet2cd
241 {
242  EIGEN_STRONG_INLINE Packet2cd() {}
243  EIGEN_STRONG_INLINE explicit Packet2cd(const __m256d& a) : v(a) {}
244  __m256d v;
245 };
246 
247 template<> struct packet_traits<std::complex<double> > : default_packet_traits
248 {
249  typedef Packet2cd type;
250  typedef Packet1cd half;
251  enum {
252  Vectorizable = 1,
253  AlignedOnScalar = 0,
254  size = 2,
255  HasHalfPacket = 1,
256 
257  HasAdd = 1,
258  HasSub = 1,
259  HasMul = 1,
260  HasDiv = 1,
261  HasNegate = 1,
262  HasAbs = 0,
263  HasAbs2 = 0,
264  HasMin = 0,
265  HasMax = 0,
266  HasSetLinear = 0
267  };
268 };
269 
270 template<> struct unpacket_traits<Packet2cd> { typedef std::complex<double> type; enum {size=2, alignment=Aligned32}; typedef Packet1cd half; };
271 
272 template<> EIGEN_STRONG_INLINE Packet2cd padd<Packet2cd>(const Packet2cd& a, const Packet2cd& b) { return Packet2cd(_mm256_add_pd(a.v,b.v)); }
273 template<> EIGEN_STRONG_INLINE Packet2cd psub<Packet2cd>(const Packet2cd& a, const Packet2cd& b) { return Packet2cd(_mm256_sub_pd(a.v,b.v)); }
274 template<> EIGEN_STRONG_INLINE Packet2cd pnegate(const Packet2cd& a) { return Packet2cd(pnegate(a.v)); }
275 template<> EIGEN_STRONG_INLINE Packet2cd pconj(const Packet2cd& a)
276 {
277  const __m256d mask = _mm256_castsi256_pd(_mm256_set_epi32(0x80000000,0x0,0x0,0x0,0x80000000,0x0,0x0,0x0));
278  return Packet2cd(_mm256_xor_pd(a.v,mask));
279 }
280 
281 template<> EIGEN_STRONG_INLINE Packet2cd pmul<Packet2cd>(const Packet2cd& a, const Packet2cd& b)
282 {
283  __m256d tmp1 = _mm256_shuffle_pd(a.v,a.v,0x0);
284  __m256d even = _mm256_mul_pd(tmp1, b.v);
285  __m256d tmp2 = _mm256_shuffle_pd(a.v,a.v,0xF);
286  __m256d tmp3 = _mm256_shuffle_pd(b.v,b.v,0x5);
287  __m256d odd = _mm256_mul_pd(tmp2, tmp3);
288  return Packet2cd(_mm256_addsub_pd(even, odd));
289 }
290 
291 template<> EIGEN_STRONG_INLINE Packet2cd pand <Packet2cd>(const Packet2cd& a, const Packet2cd& b) { return Packet2cd(_mm256_and_pd(a.v,b.v)); }
292 template<> EIGEN_STRONG_INLINE Packet2cd por <Packet2cd>(const Packet2cd& a, const Packet2cd& b) { return Packet2cd(_mm256_or_pd(a.v,b.v)); }
293 template<> EIGEN_STRONG_INLINE Packet2cd pxor <Packet2cd>(const Packet2cd& a, const Packet2cd& b) { return Packet2cd(_mm256_xor_pd(a.v,b.v)); }
294 template<> EIGEN_STRONG_INLINE Packet2cd pandnot<Packet2cd>(const Packet2cd& a, const Packet2cd& b) { return Packet2cd(_mm256_andnot_pd(a.v,b.v)); }
295 
296 template<> EIGEN_STRONG_INLINE Packet2cd pload <Packet2cd>(const std::complex<double>* from)
297 { EIGEN_DEBUG_ALIGNED_LOAD return Packet2cd(pload<Packet4d>((const double*)from)); }
298 template<> EIGEN_STRONG_INLINE Packet2cd ploadu<Packet2cd>(const std::complex<double>* from)
299 { EIGEN_DEBUG_UNALIGNED_LOAD return Packet2cd(ploadu<Packet4d>((const double*)from)); }
300 
301 template<> EIGEN_STRONG_INLINE Packet2cd pset1<Packet2cd>(const std::complex<double>& from)
302 {
303  // in case casting to a __m128d* is really not safe, then we can still fallback to this version: (much slower though)
304 // return Packet2cd(_mm256_loadu2_m128d((const double*)&from,(const double*)&from));
305  return Packet2cd(_mm256_broadcast_pd((const __m128d*)(const void*)&from));
306 }
307 
308 template<> EIGEN_STRONG_INLINE Packet2cd ploaddup<Packet2cd>(const std::complex<double>* from) { return pset1<Packet2cd>(*from); }
309 
310 template<> EIGEN_STRONG_INLINE void pstore <std::complex<double> >(std::complex<double> * to, const Packet2cd& from) { EIGEN_DEBUG_ALIGNED_STORE pstore((double*)to, from.v); }
311 template<> EIGEN_STRONG_INLINE void pstoreu<std::complex<double> >(std::complex<double> * to, const Packet2cd& from) { EIGEN_DEBUG_UNALIGNED_STORE pstoreu((double*)to, from.v); }
312 
313 template<> EIGEN_DEVICE_FUNC inline Packet2cd pgather<std::complex<double>, Packet2cd>(const std::complex<double>* from, Index stride)
314 {
315  return Packet2cd(_mm256_set_pd(std::imag(from[1*stride]), std::real(from[1*stride]),
316  std::imag(from[0*stride]), std::real(from[0*stride])));
317 }
318 
319 template<> EIGEN_DEVICE_FUNC inline void pscatter<std::complex<double>, Packet2cd>(std::complex<double>* to, const Packet2cd& from, Index stride)
320 {
321  __m128d low = _mm256_extractf128_pd(from.v, 0);
322  to[stride*0] = std::complex<double>(_mm_cvtsd_f64(low), _mm_cvtsd_f64(_mm_shuffle_pd(low, low, 1)));
323  __m128d high = _mm256_extractf128_pd(from.v, 1);
324  to[stride*1] = std::complex<double>(_mm_cvtsd_f64(high), _mm_cvtsd_f64(_mm_shuffle_pd(high, high, 1)));
325 }
326 
327 template<> EIGEN_STRONG_INLINE std::complex<double> pfirst<Packet2cd>(const Packet2cd& a)
328 {
329  __m128d low = _mm256_extractf128_pd(a.v, 0);
330  EIGEN_ALIGN16 double res[2];
331  _mm_store_pd(res, low);
332  return std::complex<double>(res[0],res[1]);
333 }
334 
335 template<> EIGEN_STRONG_INLINE Packet2cd preverse(const Packet2cd& a) {
336  __m256d result = _mm256_permute2f128_pd(a.v, a.v, 1);
337  return Packet2cd(result);
338 }
339 
340 template<> EIGEN_STRONG_INLINE std::complex<double> predux<Packet2cd>(const Packet2cd& a)
341 {
342  return predux(padd(Packet1cd(_mm256_extractf128_pd(a.v,0)),
343  Packet1cd(_mm256_extractf128_pd(a.v,1))));
344 }
345 
346 template<> EIGEN_STRONG_INLINE Packet2cd preduxp<Packet2cd>(const Packet2cd* vecs)
347 {
348  Packet4d t0 = _mm256_permute2f128_pd(vecs[0].v,vecs[1].v, 0 + (2<<4));
349  Packet4d t1 = _mm256_permute2f128_pd(vecs[0].v,vecs[1].v, 1 + (3<<4));
350 
351  return Packet2cd(_mm256_add_pd(t0,t1));
352 }
353 
354 template<> EIGEN_STRONG_INLINE std::complex<double> predux_mul<Packet2cd>(const Packet2cd& a)
355 {
356  return predux(pmul(Packet1cd(_mm256_extractf128_pd(a.v,0)),
357  Packet1cd(_mm256_extractf128_pd(a.v,1))));
358 }
359 
360 template<int Offset>
361 struct palign_impl<Offset,Packet2cd>
362 {
363  static EIGEN_STRONG_INLINE void run(Packet2cd& first, const Packet2cd& second)
364  {
365  if (Offset==0) return;
366  palign_impl<Offset*2,Packet4d>::run(first.v, second.v);
367  }
368 };
369 
370 template<> struct conj_helper<Packet2cd, Packet2cd, false,true>
371 {
372  EIGEN_STRONG_INLINE Packet2cd pmadd(const Packet2cd& x, const Packet2cd& y, const Packet2cd& c) const
373  { return padd(pmul(x,y),c); }
374 
375  EIGEN_STRONG_INLINE Packet2cd pmul(const Packet2cd& a, const Packet2cd& b) const
376  {
377  return internal::pmul(a, pconj(b));
378  }
379 };
380 
381 template<> struct conj_helper<Packet2cd, Packet2cd, true,false>
382 {
383  EIGEN_STRONG_INLINE Packet2cd pmadd(const Packet2cd& x, const Packet2cd& y, const Packet2cd& c) const
384  { return padd(pmul(x,y),c); }
385 
386  EIGEN_STRONG_INLINE Packet2cd pmul(const Packet2cd& a, const Packet2cd& b) const
387  {
388  return internal::pmul(pconj(a), b);
389  }
390 };
391 
392 template<> struct conj_helper<Packet2cd, Packet2cd, true,true>
393 {
394  EIGEN_STRONG_INLINE Packet2cd pmadd(const Packet2cd& x, const Packet2cd& y, const Packet2cd& c) const
395  { return padd(pmul(x,y),c); }
396 
397  EIGEN_STRONG_INLINE Packet2cd pmul(const Packet2cd& a, const Packet2cd& b) const
398  {
399  return pconj(internal::pmul(a, b));
400  }
401 };
402 
403 template<> struct conj_helper<Packet4d, Packet2cd, false,false>
404 {
405  EIGEN_STRONG_INLINE Packet2cd pmadd(const Packet4d& x, const Packet2cd& y, const Packet2cd& c) const
406  { return padd(c, pmul(x,y)); }
407 
408  EIGEN_STRONG_INLINE Packet2cd pmul(const Packet4d& x, const Packet2cd& y) const
409  { return Packet2cd(Eigen::internal::pmul(x, y.v)); }
410 };
411 
412 template<> struct conj_helper<Packet2cd, Packet4d, false,false>
413 {
414  EIGEN_STRONG_INLINE Packet2cd pmadd(const Packet2cd& x, const Packet4d& y, const Packet2cd& c) const
415  { return padd(c, pmul(x,y)); }
416 
417  EIGEN_STRONG_INLINE Packet2cd pmul(const Packet2cd& x, const Packet4d& y) const
418  { return Packet2cd(Eigen::internal::pmul(x.v, y)); }
419 };
420 
421 template<> EIGEN_STRONG_INLINE Packet2cd pdiv<Packet2cd>(const Packet2cd& a, const Packet2cd& b)
422 {
423  Packet2cd num = pmul(a, pconj(b));
424  __m256d tmp = _mm256_mul_pd(b.v, b.v);
425  __m256d denom = _mm256_hadd_pd(tmp, tmp);
426  return Packet2cd(_mm256_div_pd(num.v, denom));
427 }
428 
429 template<> EIGEN_STRONG_INLINE Packet2cd pcplxflip<Packet2cd>(const Packet2cd& x)
430 {
431  return Packet2cd(_mm256_shuffle_pd(x.v, x.v, 0x5));
432 }
433 
434 EIGEN_DEVICE_FUNC inline void
435 ptranspose(PacketBlock<Packet4cf,4>& kernel) {
436  __m256d P0 = _mm256_castps_pd(kernel.packet[0].v);
437  __m256d P1 = _mm256_castps_pd(kernel.packet[1].v);
438  __m256d P2 = _mm256_castps_pd(kernel.packet[2].v);
439  __m256d P3 = _mm256_castps_pd(kernel.packet[3].v);
440 
441  __m256d T0 = _mm256_shuffle_pd(P0, P1, 15);
442  __m256d T1 = _mm256_shuffle_pd(P0, P1, 0);
443  __m256d T2 = _mm256_shuffle_pd(P2, P3, 15);
444  __m256d T3 = _mm256_shuffle_pd(P2, P3, 0);
445 
446  kernel.packet[1].v = _mm256_castpd_ps(_mm256_permute2f128_pd(T0, T2, 32));
447  kernel.packet[3].v = _mm256_castpd_ps(_mm256_permute2f128_pd(T0, T2, 49));
448  kernel.packet[0].v = _mm256_castpd_ps(_mm256_permute2f128_pd(T1, T3, 32));
449  kernel.packet[2].v = _mm256_castpd_ps(_mm256_permute2f128_pd(T1, T3, 49));
450 }
451 
452 EIGEN_DEVICE_FUNC inline void
453 ptranspose(PacketBlock<Packet2cd,2>& kernel) {
454  __m256d tmp = _mm256_permute2f128_pd(kernel.packet[0].v, kernel.packet[1].v, 0+(2<<4));
455  kernel.packet[1].v = _mm256_permute2f128_pd(kernel.packet[0].v, kernel.packet[1].v, 1+(3<<4));
456  kernel.packet[0].v = tmp;
457 }
458 
459 } // end namespace internal
460 
461 } // end namespace Eigen
462 
463 #endif // EIGEN_COMPLEX_AVX_H
Eigen
Namespace containing all symbols from the Eigen library.
Definition: Core:271
std
Definition: Half.h:502
Eigen::Aligned32
Definition: Constants.h:231
Eigen::Index
EIGEN_DEFAULT_DENSE_INDEX_TYPE Index
The Index type as used for the API.
Definition: XprHelper.h:35
internal
Definition: Eigen_Colamd.h:50