13 #ifndef EIGEN_PRODUCTEVALUATORS_H 14 #define EIGEN_PRODUCTEVALUATORS_H 28 template<
typename Lhs,
typename Rhs,
int Options>
29 struct evaluator<Product<Lhs, Rhs, Options> >
30 :
public product_evaluator<Product<Lhs, Rhs, Options> >
32 typedef Product<Lhs, Rhs, Options> XprType;
33 typedef product_evaluator<XprType> Base;
35 EIGEN_DEVICE_FUNC
explicit evaluator(
const XprType& xpr) : Base(xpr) {}
40 template<
typename Lhs,
typename Rhs,
typename Scalar1,
typename Scalar2,
typename Plain1>
41 struct evaluator_assume_aliasing<CwiseBinaryOp<
internal::scalar_product_op<Scalar1,Scalar2>,
42 const CwiseNullaryOp<internal::scalar_constant_op<Scalar1>, Plain1>,
43 const Product<Lhs, Rhs, DefaultProduct> > >
45 static const bool value =
true;
47 template<
typename Lhs,
typename Rhs,
typename Scalar1,
typename Scalar2,
typename Plain1>
48 struct evaluator<CwiseBinaryOp<
internal::scalar_product_op<Scalar1,Scalar2>,
49 const CwiseNullaryOp<internal::scalar_constant_op<Scalar1>, Plain1>,
50 const Product<Lhs, Rhs, DefaultProduct> > >
51 :
public evaluator<Product<EIGEN_SCALAR_BINARYOP_EXPR_RETURN_TYPE(Scalar1,Lhs,product), Rhs, DefaultProduct> >
53 typedef CwiseBinaryOp<internal::scalar_product_op<Scalar1,Scalar2>,
54 const CwiseNullaryOp<internal::scalar_constant_op<Scalar1>, Plain1>,
55 const Product<Lhs, Rhs, DefaultProduct> > XprType;
56 typedef evaluator<Product<EIGEN_SCALAR_BINARYOP_EXPR_RETURN_TYPE(Scalar1,Lhs,product), Rhs, DefaultProduct> > Base;
58 EIGEN_DEVICE_FUNC
explicit evaluator(
const XprType& xpr)
59 : Base(xpr.lhs().functor().m_other * xpr.rhs().lhs() * xpr.rhs().rhs())
64 template<
typename Lhs,
typename Rhs,
int DiagIndex>
65 struct evaluator<Diagonal<const Product<Lhs, Rhs, DefaultProduct>, DiagIndex> >
66 :
public evaluator<Diagonal<const Product<Lhs, Rhs, LazyProduct>, DiagIndex> >
68 typedef Diagonal<const Product<Lhs, Rhs, DefaultProduct>, DiagIndex> XprType;
69 typedef evaluator<Diagonal<const Product<Lhs, Rhs, LazyProduct>, DiagIndex> > Base;
71 EIGEN_DEVICE_FUNC
explicit evaluator(
const XprType& xpr)
72 : Base(Diagonal<const Product<Lhs, Rhs, LazyProduct>, DiagIndex>(
73 Product<Lhs, Rhs, LazyProduct>(xpr.nestedExpression().lhs(), xpr.nestedExpression().rhs()),
82 template<
typename Lhs,
typename Rhs,
83 typename LhsShape =
typename evaluator_traits<Lhs>::Shape,
84 typename RhsShape =
typename evaluator_traits<Rhs>::Shape,
85 int ProductType = internal::product_type<Lhs,Rhs>::value>
86 struct generic_product_impl;
88 template<
typename Lhs,
typename Rhs>
89 struct evaluator_assume_aliasing<Product<Lhs, Rhs, DefaultProduct> > {
90 static const bool value =
true;
95 template<
typename Lhs,
typename Rhs,
int Options,
int ProductTag,
typename LhsShape,
typename RhsShape>
96 struct product_evaluator<Product<Lhs, Rhs, Options>, ProductTag, LhsShape, RhsShape>
97 :
public evaluator<typename Product<Lhs, Rhs, Options>::PlainObject>
99 typedef Product<Lhs, Rhs, Options> XprType;
100 typedef typename XprType::PlainObject PlainObject;
101 typedef evaluator<PlainObject> Base;
106 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
107 explicit product_evaluator(
const XprType& xpr)
108 : m_result(xpr.rows(), xpr.cols())
110 ::new (static_cast<Base*>(
this)) Base(m_result);
124 generic_product_impl<Lhs, Rhs, LhsShape, RhsShape, ProductTag>::evalTo(m_result, xpr.lhs(), xpr.rhs());
128 PlainObject m_result;
135 template< typename DstXprType, typename Lhs, typename Rhs,
int Options, typename Scalar>
136 struct Assignment<DstXprType, Product<Lhs,Rhs,Options>,
internal::assign_op<Scalar,Scalar>, Dense2Dense,
137 typename enable_if<(Options==DefaultProduct || Options==AliasFreeProduct)>::type>
139 typedef Product<Lhs,Rhs,Options> SrcXprType;
140 static EIGEN_STRONG_INLINE
141 void run(DstXprType &dst,
const SrcXprType &src,
const internal::assign_op<Scalar,Scalar> &)
144 generic_product_impl<Lhs, Rhs>::evalTo(dst, src.lhs(), src.rhs());
149 template<
typename DstXprType,
typename Lhs,
typename Rhs,
int Options,
typename Scalar>
150 struct Assignment<DstXprType, Product<Lhs,Rhs,Options>,
internal::add_assign_op<Scalar,Scalar>, Dense2Dense,
151 typename enable_if<(Options==DefaultProduct || Options==AliasFreeProduct)>::type>
153 typedef Product<Lhs,Rhs,Options> SrcXprType;
154 static EIGEN_STRONG_INLINE
155 void run(DstXprType &dst,
const SrcXprType &src,
const internal::add_assign_op<Scalar,Scalar> &)
158 generic_product_impl<Lhs, Rhs>::addTo(dst, src.lhs(), src.rhs());
163 template<
typename DstXprType,
typename Lhs,
typename Rhs,
int Options,
typename Scalar>
164 struct Assignment<DstXprType, Product<Lhs,Rhs,Options>,
internal::sub_assign_op<Scalar,Scalar>, Dense2Dense,
165 typename enable_if<(Options==DefaultProduct || Options==AliasFreeProduct)>::type>
167 typedef Product<Lhs,Rhs,Options> SrcXprType;
168 static EIGEN_STRONG_INLINE
169 void run(DstXprType &dst,
const SrcXprType &src,
const internal::sub_assign_op<Scalar,Scalar> &)
172 generic_product_impl<Lhs, Rhs>::subTo(dst, src.lhs(), src.rhs());
180 template<
typename DstXprType,
typename Lhs,
typename Rhs,
typename AssignFunc,
typename Scalar,
typename ScalarBis,
typename Plain>
181 struct Assignment<DstXprType, CwiseBinaryOp<
internal::scalar_product_op<ScalarBis,Scalar>, const CwiseNullaryOp<internal::scalar_constant_op<ScalarBis>,Plain>,
182 const Product<Lhs,Rhs,DefaultProduct> >, AssignFunc, Dense2Dense>
184 typedef CwiseBinaryOp<internal::scalar_product_op<ScalarBis,Scalar>,
185 const CwiseNullaryOp<internal::scalar_constant_op<ScalarBis>,Plain>,
186 const Product<Lhs,Rhs,DefaultProduct> > SrcXprType;
187 static EIGEN_STRONG_INLINE
188 void run(DstXprType &dst,
const SrcXprType &src,
const AssignFunc& func)
190 call_assignment_no_alias(dst, (src.lhs().functor().m_other * src.rhs().lhs())*src.rhs().rhs(), func);
199 template<
typename OtherXpr,
typename Lhs,
typename Rhs>
200 struct evaluator_assume_aliasing<CwiseBinaryOp<
internal::scalar_sum_op<typename OtherXpr::Scalar,typename Product<Lhs,Rhs,DefaultProduct>::Scalar>, const OtherXpr,
201 const Product<Lhs,Rhs,DefaultProduct> >, DenseShape > {
202 static const bool value =
true;
205 template<
typename DstXprType,
typename OtherXpr,
typename ProductType,
typename Func1,
typename Func2>
206 struct assignment_from_xpr_plus_product
208 typedef CwiseBinaryOp<internal::scalar_sum_op<typename OtherXpr::Scalar,typename ProductType::Scalar>,
const OtherXpr,
const ProductType> SrcXprType;
209 template<
typename InitialFunc>
210 static EIGEN_STRONG_INLINE
211 void run(DstXprType &dst,
const SrcXprType &src,
const InitialFunc& )
213 call_assignment_no_alias(dst, src.lhs(), Func1());
214 call_assignment_no_alias(dst, src.rhs(), Func2());
218 template<
typename DstXprType,
typename OtherXpr,
typename Lhs,
typename Rhs,
typename DstScalar,
typename SrcScalar,
typename OtherScalar,
typename ProdScalar>
219 struct Assignment<DstXprType, CwiseBinaryOp<
internal::scalar_sum_op<OtherScalar,ProdScalar>, const OtherXpr,
220 const Product<Lhs,Rhs,DefaultProduct> >, internal::assign_op<DstScalar,SrcScalar>, Dense2Dense>
221 : assignment_from_xpr_plus_product<DstXprType, OtherXpr, Product<Lhs,Rhs,DefaultProduct>, internal::assign_op<DstScalar,OtherScalar>, internal::add_assign_op<DstScalar,ProdScalar> >
223 template<
typename DstXprType,
typename OtherXpr,
typename Lhs,
typename Rhs,
typename DstScalar,
typename SrcScalar,
typename OtherScalar,
typename ProdScalar>
224 struct Assignment<DstXprType, CwiseBinaryOp<internal::scalar_sum_op<OtherScalar,ProdScalar>, const OtherXpr,
225 const Product<Lhs,Rhs,DefaultProduct> >, internal::add_assign_op<DstScalar,SrcScalar>, Dense2Dense>
226 : assignment_from_xpr_plus_product<DstXprType, OtherXpr, Product<Lhs,Rhs,DefaultProduct>, internal::add_assign_op<DstScalar,OtherScalar>, internal::add_assign_op<DstScalar,ProdScalar> >
228 template<
typename DstXprType,
typename OtherXpr,
typename Lhs,
typename Rhs,
typename DstScalar,
typename SrcScalar,
typename OtherScalar,
typename ProdScalar>
229 struct Assignment<DstXprType, CwiseBinaryOp<internal::scalar_sum_op<OtherScalar,ProdScalar>, const OtherXpr,
230 const Product<Lhs,Rhs,DefaultProduct> >, internal::sub_assign_op<DstScalar,SrcScalar>, Dense2Dense>
231 : assignment_from_xpr_plus_product<DstXprType, OtherXpr, Product<Lhs,Rhs,DefaultProduct>, internal::sub_assign_op<DstScalar,OtherScalar>, internal::sub_assign_op<DstScalar,ProdScalar> >
235 template<
typename Lhs,
typename Rhs>
236 struct generic_product_impl<Lhs,Rhs,DenseShape,DenseShape,InnerProduct>
238 template<
typename Dst>
239 static inline void evalTo(Dst& dst,
const Lhs& lhs,
const Rhs& rhs)
241 dst.coeffRef(0,0) = (lhs.transpose().cwiseProduct(rhs)).sum();
244 template<
typename Dst>
245 static inline void addTo(Dst& dst,
const Lhs& lhs,
const Rhs& rhs)
247 dst.coeffRef(0,0) += (lhs.transpose().cwiseProduct(rhs)).sum();
250 template<
typename Dst>
251 static void subTo(Dst& dst,
const Lhs& lhs,
const Rhs& rhs)
252 { dst.coeffRef(0,0) -= (lhs.transpose().cwiseProduct(rhs)).sum(); }
261 template<
typename Dst,
typename Lhs,
typename Rhs,
typename Func>
262 void outer_product_selector_run(Dst& dst,
const Lhs &lhs,
const Rhs &rhs,
const Func& func,
const false_type&)
264 evaluator<Rhs> rhsEval(rhs);
265 typename nested_eval<Lhs,Rhs::SizeAtCompileTime>::type actual_lhs(lhs);
268 const Index cols = dst.cols();
269 for (
Index j=0; j<cols; ++j)
270 func(dst.col(j), rhsEval.coeff(0,j) * actual_lhs);
274 template<
typename Dst,
typename Lhs,
typename Rhs,
typename Func>
275 void outer_product_selector_run(Dst& dst,
const Lhs &lhs,
const Rhs &rhs,
const Func& func,
const true_type&)
277 evaluator<Lhs> lhsEval(lhs);
278 typename nested_eval<Rhs,Lhs::SizeAtCompileTime>::type actual_rhs(rhs);
281 const Index rows = dst.rows();
282 for (
Index i=0; i<rows; ++i)
283 func(dst.row(i), lhsEval.coeff(i,0) * actual_rhs);
286 template<
typename Lhs,
typename Rhs>
287 struct generic_product_impl<Lhs,Rhs,DenseShape,DenseShape,OuterProduct>
289 template<
typename T>
struct is_row_major : internal::conditional<(int(T::Flags)&RowMajorBit), internal::true_type, internal::false_type>::type {};
290 typedef typename Product<Lhs,Rhs>::Scalar Scalar;
293 struct set {
template<
typename Dst,
typename Src>
void operator()(
const Dst& dst,
const Src& src)
const { dst.const_cast_derived() = src; } };
294 struct add {
template<
typename Dst,
typename Src>
void operator()(
const Dst& dst,
const Src& src)
const { dst.const_cast_derived() += src; } };
295 struct sub {
template<
typename Dst,
typename Src>
void operator()(
const Dst& dst,
const Src& src)
const { dst.const_cast_derived() -= src; } };
298 explicit adds(
const Scalar& s) : m_scale(s) {}
299 template<
typename Dst,
typename Src>
void operator()(
const Dst& dst,
const Src& src)
const {
300 dst.const_cast_derived() += m_scale * src;
304 template<
typename Dst>
305 static inline void evalTo(Dst& dst,
const Lhs& lhs,
const Rhs& rhs)
307 internal::outer_product_selector_run(dst, lhs, rhs,
set(), is_row_major<Dst>());
310 template<
typename Dst>
311 static inline void addTo(Dst& dst,
const Lhs& lhs,
const Rhs& rhs)
313 internal::outer_product_selector_run(dst, lhs, rhs, add(), is_row_major<Dst>());
316 template<
typename Dst>
317 static inline void subTo(Dst& dst,
const Lhs& lhs,
const Rhs& rhs)
319 internal::outer_product_selector_run(dst, lhs, rhs, sub(), is_row_major<Dst>());
322 template<
typename Dst>
323 static inline void scaleAndAddTo(Dst& dst,
const Lhs& lhs,
const Rhs& rhs,
const Scalar& alpha)
325 internal::outer_product_selector_run(dst, lhs, rhs, adds(alpha), is_row_major<Dst>());
332 template<
typename Lhs,
typename Rhs,
typename Derived>
333 struct generic_product_impl_base
335 typedef typename Product<Lhs,Rhs>::Scalar Scalar;
337 template<
typename Dst>
338 static EIGEN_STRONG_INLINE
void evalTo(Dst& dst,
const Lhs& lhs,
const Rhs& rhs)
339 { dst.setZero(); scaleAndAddTo(dst, lhs, rhs, Scalar(1)); }
341 template<
typename Dst>
342 static EIGEN_STRONG_INLINE
void addTo(Dst& dst,
const Lhs& lhs,
const Rhs& rhs)
343 { scaleAndAddTo(dst,lhs, rhs, Scalar(1)); }
345 template<
typename Dst>
346 static EIGEN_STRONG_INLINE
void subTo(Dst& dst,
const Lhs& lhs,
const Rhs& rhs)
347 { scaleAndAddTo(dst, lhs, rhs, Scalar(-1)); }
349 template<
typename Dst>
350 static EIGEN_STRONG_INLINE
void scaleAndAddTo(Dst& dst,
const Lhs& lhs,
const Rhs& rhs,
const Scalar& alpha)
351 { Derived::scaleAndAddTo(dst,lhs,rhs,alpha); }
355 template<
typename Lhs,
typename Rhs>
356 struct generic_product_impl<Lhs,Rhs,DenseShape,DenseShape,GemvProduct>
357 : generic_product_impl_base<Lhs,Rhs,generic_product_impl<Lhs,Rhs,DenseShape,DenseShape,GemvProduct> >
359 typedef typename Product<Lhs,Rhs>::Scalar Scalar;
361 typedef typename internal::conditional<int(Side)==OnTheRight,Lhs,Rhs>::type MatrixType;
363 template<
typename Dest>
364 static EIGEN_STRONG_INLINE
void scaleAndAddTo(Dest& dst,
const Lhs& lhs,
const Rhs& rhs,
const Scalar& alpha)
366 internal::gemv_dense_selector<Side,
368 bool(internal::blas_traits<MatrixType>::HasUsableDirectAccess)
369 >::run(lhs, rhs, dst, alpha);
373 template<
typename Lhs,
typename Rhs>
374 struct generic_product_impl<Lhs,Rhs,DenseShape,DenseShape,CoeffBasedProductMode>
376 typedef typename Product<Lhs,Rhs>::Scalar Scalar;
378 template<
typename Dst>
379 static EIGEN_STRONG_INLINE
void evalTo(Dst& dst,
const Lhs& lhs,
const Rhs& rhs)
383 call_assignment_no_alias(dst, lhs.lazyProduct(rhs), internal::assign_op<typename Dst::Scalar,Scalar>());
386 template<
typename Dst>
387 static EIGEN_STRONG_INLINE
void addTo(Dst& dst,
const Lhs& lhs,
const Rhs& rhs)
390 call_assignment_no_alias(dst, lhs.lazyProduct(rhs), internal::add_assign_op<typename Dst::Scalar,Scalar>());
393 template<
typename Dst>
394 static EIGEN_STRONG_INLINE
void subTo(Dst& dst,
const Lhs& lhs,
const Rhs& rhs)
397 call_assignment_no_alias(dst, lhs.lazyProduct(rhs), internal::sub_assign_op<typename Dst::Scalar,Scalar>());
406 template<
typename Lhs,
typename Rhs>
407 struct generic_product_impl<Lhs,Rhs,DenseShape,DenseShape,LazyCoeffBasedProductMode>
408 : generic_product_impl<Lhs,Rhs,DenseShape,DenseShape,CoeffBasedProductMode> {};
416 template<
int Traversal,
int UnrollingIndex,
typename Lhs,
typename Rhs,
typename RetScalar>
417 struct etor_product_coeff_impl;
419 template<
int StorageOrder,
int UnrollingIndex,
typename Lhs,
typename Rhs,
typename Packet,
int LoadMode>
420 struct etor_product_packet_impl;
422 template<
typename Lhs,
typename Rhs,
int ProductTag>
423 struct product_evaluator<Product<Lhs, Rhs, LazyProduct>, ProductTag, DenseShape, DenseShape>
424 : evaluator_base<Product<Lhs, Rhs, LazyProduct> >
426 typedef Product<Lhs, Rhs, LazyProduct> XprType;
427 typedef typename XprType::Scalar Scalar;
428 typedef typename XprType::CoeffReturnType CoeffReturnType;
430 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
431 explicit product_evaluator(
const XprType& xpr)
437 m_innerDim(xpr.lhs().cols())
439 EIGEN_INTERNAL_CHECK_COST_VALUE(NumTraits<Scalar>::MulCost);
440 EIGEN_INTERNAL_CHECK_COST_VALUE(NumTraits<Scalar>::AddCost);
441 EIGEN_INTERNAL_CHECK_COST_VALUE(CoeffReadCost);
446 typedef typename internal::nested_eval<Lhs,Rhs::ColsAtCompileTime>::type LhsNested;
447 typedef typename internal::nested_eval<Rhs,Lhs::RowsAtCompileTime>::type RhsNested;
449 typedef typename internal::remove_all<LhsNested>::type LhsNestedCleaned;
450 typedef typename internal::remove_all<RhsNested>::type RhsNestedCleaned;
452 typedef evaluator<LhsNestedCleaned> LhsEtorType;
453 typedef evaluator<RhsNestedCleaned> RhsEtorType;
456 RowsAtCompileTime = LhsNestedCleaned::RowsAtCompileTime,
457 ColsAtCompileTime = RhsNestedCleaned::ColsAtCompileTime,
458 InnerSize = EIGEN_SIZE_MIN_PREFER_FIXED(LhsNestedCleaned::ColsAtCompileTime, RhsNestedCleaned::RowsAtCompileTime),
459 MaxRowsAtCompileTime = LhsNestedCleaned::MaxRowsAtCompileTime,
460 MaxColsAtCompileTime = RhsNestedCleaned::MaxColsAtCompileTime
463 typedef typename find_best_packet<Scalar,RowsAtCompileTime>::type LhsVecPacketType;
464 typedef typename find_best_packet<Scalar,ColsAtCompileTime>::type RhsVecPacketType;
468 LhsCoeffReadCost = LhsEtorType::CoeffReadCost,
469 RhsCoeffReadCost = RhsEtorType::CoeffReadCost,
470 CoeffReadCost = InnerSize==0 ? NumTraits<Scalar>::ReadCost
472 : InnerSize * (NumTraits<Scalar>::MulCost + LhsCoeffReadCost + RhsCoeffReadCost)
473 + (InnerSize - 1) * NumTraits<Scalar>::AddCost,
475 Unroll = CoeffReadCost <= EIGEN_UNROLLING_LIMIT,
477 LhsFlags = LhsEtorType::Flags,
478 RhsFlags = RhsEtorType::Flags,
483 LhsVecPacketSize = unpacket_traits<LhsVecPacketType>::size,
484 RhsVecPacketSize = unpacket_traits<RhsVecPacketType>::size,
487 LhsAlignment = EIGEN_PLAIN_ENUM_MIN(LhsEtorType::Alignment,LhsVecPacketSize*
int(
sizeof(
typename LhsNestedCleaned::Scalar))),
488 RhsAlignment = EIGEN_PLAIN_ENUM_MIN(RhsEtorType::Alignment,RhsVecPacketSize*
int(
sizeof(
typename RhsNestedCleaned::Scalar))),
490 SameType = is_same<typename LhsNestedCleaned::Scalar,typename RhsNestedCleaned::Scalar>::value,
495 EvalToRowMajor = (MaxRowsAtCompileTime==1&&MaxColsAtCompileTime!=1) ? 1
496 : (MaxColsAtCompileTime==1&&MaxRowsAtCompileTime!=1) ? 0
497 : (
bool(RhsRowMajor) && !CanVectorizeLhs),
499 Flags = ((
unsigned int)(LhsFlags | RhsFlags) & HereditaryBits & ~RowMajorBit)
500 | (EvalToRowMajor ? RowMajorBit : 0)
502 | (SameType && (CanVectorizeLhs || CanVectorizeRhs) ?
PacketAccessBit : 0)
505 LhsOuterStrideBytes = int(LhsNestedCleaned::OuterStrideAtCompileTime) * int(
sizeof(
typename LhsNestedCleaned::Scalar)),
506 RhsOuterStrideBytes = int(RhsNestedCleaned::OuterStrideAtCompileTime) * int(
sizeof(
typename RhsNestedCleaned::Scalar)),
508 Alignment = bool(CanVectorizeLhs) ? (LhsOuterStrideBytes<0 || (int(LhsOuterStrideBytes) % EIGEN_PLAIN_ENUM_MAX(1,LhsAlignment))!=0 ? 0 : LhsAlignment)
509 : bool(CanVectorizeRhs) ? (RhsOuterStrideBytes<0 || (int(RhsOuterStrideBytes) % EIGEN_PLAIN_ENUM_MAX(1,RhsAlignment))!=0 ? 0 : RhsAlignment)
517 CanVectorizeInner = SameType
521 && (InnerSize % packet_traits<Scalar>::size == 0)
524 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
const CoeffReturnType coeff(
Index row,
Index col)
const 526 return (m_lhs.row(row).transpose().cwiseProduct( m_rhs.col(col) )).sum();
533 EIGEN_DEVICE_FUNC
const CoeffReturnType coeff(
Index index)
const 535 const Index row = RowsAtCompileTime == 1 ? 0 : index;
536 const Index col = RowsAtCompileTime == 1 ? index : 0;
537 return (m_lhs.row(row).transpose().cwiseProduct( m_rhs.col(col) )).sum();
540 template<
int LoadMode,
typename PacketType>
541 const PacketType packet(
Index row,
Index col)
const 544 typedef etor_product_packet_impl<bool(
int(Flags)&RowMajorBit) ?
RowMajor :
ColMajor,
545 Unroll ? int(InnerSize) :
Dynamic,
546 LhsEtorType, RhsEtorType, PacketType, LoadMode> PacketImpl;
547 PacketImpl::run(row, col, m_lhsImpl, m_rhsImpl, m_innerDim, res);
551 template<
int LoadMode,
typename PacketType>
552 const PacketType packet(
Index index)
const 554 const Index row = RowsAtCompileTime == 1 ? 0 : index;
555 const Index col = RowsAtCompileTime == 1 ? index : 0;
556 return packet<LoadMode,PacketType>(row,col);
560 const LhsNested m_lhs;
561 const RhsNested m_rhs;
563 LhsEtorType m_lhsImpl;
564 RhsEtorType m_rhsImpl;
570 template<
typename Lhs,
typename Rhs>
571 struct product_evaluator<Product<Lhs, Rhs, DefaultProduct>, LazyCoeffBasedProductMode, DenseShape, DenseShape>
572 : product_evaluator<Product<Lhs, Rhs, LazyProduct>, CoeffBasedProductMode, DenseShape, DenseShape>
574 typedef Product<Lhs, Rhs, DefaultProduct> XprType;
575 typedef Product<Lhs, Rhs, LazyProduct> BaseProduct;
576 typedef product_evaluator<BaseProduct, CoeffBasedProductMode, DenseShape, DenseShape> Base;
580 EIGEN_DEVICE_FUNC
explicit product_evaluator(
const XprType& xpr)
581 : Base(BaseProduct(xpr.lhs(),xpr.rhs()))
589 template<
int UnrollingIndex,
typename Lhs,
typename Rhs,
typename Packet,
int LoadMode>
590 struct etor_product_packet_impl<
RowMajor, UnrollingIndex, Lhs, Rhs, Packet, LoadMode>
592 static EIGEN_STRONG_INLINE
void run(
Index row,
Index col,
const Lhs& lhs,
const Rhs& rhs,
Index innerDim, Packet &res)
594 etor_product_packet_impl<RowMajor, UnrollingIndex-1, Lhs, Rhs, Packet, LoadMode>::run(row, col, lhs, rhs, innerDim, res);
595 res = pmadd(pset1<Packet>(lhs.coeff(row, UnrollingIndex-1)), rhs.template packet<LoadMode,Packet>(UnrollingIndex-1, col), res);
599 template<
int UnrollingIndex,
typename Lhs,
typename Rhs,
typename Packet,
int LoadMode>
600 struct etor_product_packet_impl<
ColMajor, UnrollingIndex, Lhs, Rhs, Packet, LoadMode>
602 static EIGEN_STRONG_INLINE
void run(
Index row,
Index col,
const Lhs& lhs,
const Rhs& rhs,
Index innerDim, Packet &res)
604 etor_product_packet_impl<ColMajor, UnrollingIndex-1, Lhs, Rhs, Packet, LoadMode>::run(row, col, lhs, rhs, innerDim, res);
605 res = pmadd(lhs.template packet<LoadMode,Packet>(row, UnrollingIndex-1), pset1<Packet>(rhs.coeff(UnrollingIndex-1, col)), res);
609 template<
typename Lhs,
typename Rhs,
typename Packet,
int LoadMode>
610 struct etor_product_packet_impl<
RowMajor, 1, Lhs, Rhs, Packet, LoadMode>
612 static EIGEN_STRONG_INLINE
void run(
Index row,
Index col,
const Lhs& lhs,
const Rhs& rhs,
Index , Packet &res)
614 res = pmul(pset1<Packet>(lhs.coeff(row, 0)),rhs.template packet<LoadMode,Packet>(0, col));
618 template<
typename Lhs,
typename Rhs,
typename Packet,
int LoadMode>
619 struct etor_product_packet_impl<
ColMajor, 1, Lhs, Rhs, Packet, LoadMode>
621 static EIGEN_STRONG_INLINE
void run(
Index row,
Index col,
const Lhs& lhs,
const Rhs& rhs,
Index , Packet &res)
623 res = pmul(lhs.template packet<LoadMode,Packet>(row, 0), pset1<Packet>(rhs.coeff(0, col)));
627 template<
typename Lhs,
typename Rhs,
typename Packet,
int LoadMode>
628 struct etor_product_packet_impl<
RowMajor, 0, Lhs, Rhs, Packet, LoadMode>
630 static EIGEN_STRONG_INLINE
void run(
Index ,
Index ,
const Lhs& ,
const Rhs& ,
Index , Packet &res)
632 res = pset1<Packet>(0);
636 template<
typename Lhs,
typename Rhs,
typename Packet,
int LoadMode>
637 struct etor_product_packet_impl<
ColMajor, 0, Lhs, Rhs, Packet, LoadMode>
639 static EIGEN_STRONG_INLINE
void run(
Index ,
Index ,
const Lhs& ,
const Rhs& ,
Index , Packet &res)
641 res = pset1<Packet>(0);
645 template<
typename Lhs,
typename Rhs,
typename Packet,
int LoadMode>
646 struct etor_product_packet_impl<
RowMajor,
Dynamic, Lhs, Rhs, Packet, LoadMode>
648 static EIGEN_STRONG_INLINE
void run(
Index row,
Index col,
const Lhs& lhs,
const Rhs& rhs,
Index innerDim, Packet& res)
650 res = pset1<Packet>(0);
651 for(
Index i = 0; i < innerDim; ++i)
652 res = pmadd(pset1<Packet>(lhs.coeff(row, i)), rhs.template packet<LoadMode,Packet>(i, col), res);
656 template<
typename Lhs,
typename Rhs,
typename Packet,
int LoadMode>
657 struct etor_product_packet_impl<
ColMajor,
Dynamic, Lhs, Rhs, Packet, LoadMode>
659 static EIGEN_STRONG_INLINE
void run(
Index row,
Index col,
const Lhs& lhs,
const Rhs& rhs,
Index innerDim, Packet& res)
661 res = pset1<Packet>(0);
662 for(
Index i = 0; i < innerDim; ++i)
663 res = pmadd(lhs.template packet<LoadMode,Packet>(row, i), pset1<Packet>(rhs.coeff(i, col)), res);
671 template<
int Mode,
bool LhsIsTriangular,
672 typename Lhs,
bool LhsIsVector,
673 typename Rhs,
bool RhsIsVector>
674 struct triangular_product_impl;
676 template<
typename Lhs,
typename Rhs,
int ProductTag>
677 struct generic_product_impl<Lhs,Rhs,TriangularShape,DenseShape,ProductTag>
678 : generic_product_impl_base<Lhs,Rhs,generic_product_impl<Lhs,Rhs,TriangularShape,DenseShape,ProductTag> >
680 typedef typename Product<Lhs,Rhs>::Scalar Scalar;
682 template<
typename Dest>
683 static void scaleAndAddTo(Dest& dst,
const Lhs& lhs,
const Rhs& rhs,
const Scalar& alpha)
685 triangular_product_impl<Lhs::Mode,true,typename Lhs::MatrixType,false,Rhs, Rhs::ColsAtCompileTime==1>
686 ::run(dst, lhs.nestedExpression(), rhs, alpha);
690 template<
typename Lhs,
typename Rhs,
int ProductTag>
691 struct generic_product_impl<Lhs,Rhs,DenseShape,TriangularShape,ProductTag>
692 : generic_product_impl_base<Lhs,Rhs,generic_product_impl<Lhs,Rhs,DenseShape,TriangularShape,ProductTag> >
694 typedef typename Product<Lhs,Rhs>::Scalar Scalar;
696 template<
typename Dest>
697 static void scaleAndAddTo(Dest& dst,
const Lhs& lhs,
const Rhs& rhs,
const Scalar& alpha)
699 triangular_product_impl<Rhs::Mode,false,Lhs,Lhs::RowsAtCompileTime==1, typename Rhs::MatrixType, false>::run(dst, lhs, rhs.nestedExpression(), alpha);
707 template <
typename Lhs,
int LhsMode,
bool LhsIsVector,
708 typename Rhs,
int RhsMode,
bool RhsIsVector>
709 struct selfadjoint_product_impl;
711 template<
typename Lhs,
typename Rhs,
int ProductTag>
712 struct generic_product_impl<Lhs,Rhs,SelfAdjointShape,DenseShape,ProductTag>
713 : generic_product_impl_base<Lhs,Rhs,generic_product_impl<Lhs,Rhs,SelfAdjointShape,DenseShape,ProductTag> >
715 typedef typename Product<Lhs,Rhs>::Scalar Scalar;
717 template<
typename Dest>
718 static void scaleAndAddTo(Dest& dst,
const Lhs& lhs,
const Rhs& rhs,
const Scalar& alpha)
720 selfadjoint_product_impl<typename Lhs::MatrixType,Lhs::Mode,false,Rhs,0,Rhs::IsVectorAtCompileTime>::run(dst, lhs.nestedExpression(), rhs, alpha);
724 template<
typename Lhs,
typename Rhs,
int ProductTag>
725 struct generic_product_impl<Lhs,Rhs,DenseShape,SelfAdjointShape,ProductTag>
726 : generic_product_impl_base<Lhs,Rhs,generic_product_impl<Lhs,Rhs,DenseShape,SelfAdjointShape,ProductTag> >
728 typedef typename Product<Lhs,Rhs>::Scalar Scalar;
730 template<
typename Dest>
731 static void scaleAndAddTo(Dest& dst,
const Lhs& lhs,
const Rhs& rhs,
const Scalar& alpha)
733 selfadjoint_product_impl<Lhs,0,Lhs::IsVectorAtCompileTime,typename Rhs::MatrixType,Rhs::Mode,false>::run(dst, lhs, rhs.nestedExpression(), alpha);
742 template<
typename MatrixType,
typename DiagonalType,
typename Derived,
int ProductOrder>
743 struct diagonal_product_evaluator_base
744 : evaluator_base<Derived>
746 typedef typename ScalarBinaryOpTraits<typename MatrixType::Scalar, typename DiagonalType::Scalar>::ReturnType Scalar;
749 CoeffReadCost = NumTraits<Scalar>::MulCost + evaluator<MatrixType>::CoeffReadCost + evaluator<DiagonalType>::CoeffReadCost,
751 MatrixFlags = evaluator<MatrixType>::Flags,
752 DiagFlags = evaluator<DiagonalType>::Flags,
754 _ScalarAccessOnDiag = !((int(_StorageOrder) ==
ColMajor && int(ProductOrder) ==
OnTheLeft)
756 _SameTypes = is_same<typename MatrixType::Scalar, typename DiagonalType::Scalar>::value,
760 _LinearAccessMask = (MatrixType::RowsAtCompileTime==1 || MatrixType::ColsAtCompileTime==1) ? LinearAccessBit : 0,
761 Flags = ((HereditaryBits|_LinearAccessMask) & (
unsigned int)(MatrixFlags)) | (_Vectorizable ?
PacketAccessBit : 0),
762 Alignment = evaluator<MatrixType>::Alignment
765 diagonal_product_evaluator_base(
const MatrixType &mat,
const DiagonalType &diag)
766 : m_diagImpl(diag), m_matImpl(mat)
768 EIGEN_INTERNAL_CHECK_COST_VALUE(NumTraits<Scalar>::MulCost);
769 EIGEN_INTERNAL_CHECK_COST_VALUE(CoeffReadCost);
772 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
const Scalar coeff(
Index idx)
const 774 return m_diagImpl.coeff(idx) * m_matImpl.coeff(idx);
778 template<
int LoadMode,
typename PacketType>
779 EIGEN_STRONG_INLINE PacketType packet_impl(
Index row,
Index col,
Index id, internal::true_type)
const 781 return internal::pmul(m_matImpl.template packet<LoadMode,PacketType>(row, col),
782 internal::pset1<PacketType>(m_diagImpl.coeff(
id)));
785 template<
int LoadMode,
typename PacketType>
786 EIGEN_STRONG_INLINE PacketType packet_impl(
Index row,
Index col,
Index id, internal::false_type)
const 789 InnerSize = (MatrixType::Flags &
RowMajorBit) ? MatrixType::ColsAtCompileTime : MatrixType::RowsAtCompileTime,
790 DiagonalPacketLoadMode = EIGEN_PLAIN_ENUM_MIN(LoadMode,((InnerSize%16) == 0) ? int(
Aligned16) : int(evaluator<DiagonalType>::Alignment))
792 return internal::pmul(m_matImpl.template packet<LoadMode,PacketType>(row, col),
793 m_diagImpl.template packet<DiagonalPacketLoadMode,PacketType>(
id));
796 evaluator<DiagonalType> m_diagImpl;
797 evaluator<MatrixType> m_matImpl;
801 template<
typename Lhs,
typename Rhs,
int ProductKind,
int ProductTag>
802 struct product_evaluator<Product<Lhs, Rhs, ProductKind>, ProductTag, DiagonalShape, DenseShape>
803 : diagonal_product_evaluator_base<Rhs, typename Lhs::DiagonalVectorType, Product<Lhs, Rhs, LazyProduct>, OnTheLeft>
805 typedef diagonal_product_evaluator_base<Rhs, typename Lhs::DiagonalVectorType, Product<Lhs, Rhs, LazyProduct>,
OnTheLeft> Base;
806 using Base::m_diagImpl;
807 using Base::m_matImpl;
809 typedef typename Base::Scalar Scalar;
811 typedef Product<Lhs, Rhs, ProductKind> XprType;
812 typedef typename XprType::PlainObject PlainObject;
818 EIGEN_DEVICE_FUNC
explicit product_evaluator(
const XprType& xpr)
819 : Base(xpr.rhs(), xpr.lhs().diagonal())
823 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
const Scalar coeff(
Index row,
Index col)
const 825 return m_diagImpl.coeff(row) * m_matImpl.coeff(row, col);
829 template<
int LoadMode,
typename PacketType>
830 EIGEN_STRONG_INLINE PacketType packet(
Index row,
Index col)
const 834 return this->
template packet_impl<LoadMode,PacketType>(row,col, row,
835 typename internal::conditional<int(StorageOrder)==RowMajor, internal::true_type, internal::false_type>::type());
838 template<
int LoadMode,
typename PacketType>
839 EIGEN_STRONG_INLINE PacketType packet(
Index idx)
const 841 return packet<LoadMode,PacketType>(int(StorageOrder)==
ColMajor?idx:0,int(StorageOrder)==
ColMajor?0:idx);
847 template<
typename Lhs,
typename Rhs,
int ProductKind,
int ProductTag>
848 struct product_evaluator<Product<Lhs, Rhs, ProductKind>, ProductTag, DenseShape, DiagonalShape>
849 : diagonal_product_evaluator_base<Lhs, typename Rhs::DiagonalVectorType, Product<Lhs, Rhs, LazyProduct>,
OnTheRight>
851 typedef diagonal_product_evaluator_base<Lhs, typename Rhs::DiagonalVectorType, Product<Lhs, Rhs, LazyProduct>,
OnTheRight> Base;
852 using Base::m_diagImpl;
853 using Base::m_matImpl;
855 typedef typename Base::Scalar Scalar;
857 typedef Product<Lhs, Rhs, ProductKind> XprType;
858 typedef typename XprType::PlainObject PlainObject;
860 enum { StorageOrder = int(Lhs::Flags) & RowMajorBit ?
RowMajor :
ColMajor };
862 EIGEN_DEVICE_FUNC
explicit product_evaluator(
const XprType& xpr)
863 : Base(xpr.lhs(), xpr.rhs().diagonal())
867 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
const Scalar coeff(
Index row,
Index col)
const 869 return m_matImpl.coeff(row, col) * m_diagImpl.coeff(col);
873 template<
int LoadMode,
typename PacketType>
874 EIGEN_STRONG_INLINE PacketType packet(
Index row,
Index col)
const 876 return this->
template packet_impl<LoadMode,PacketType>(row,col, col,
877 typename internal::conditional<int(StorageOrder)==ColMajor, internal::true_type, internal::false_type>::type());
880 template<
int LoadMode,
typename PacketType>
881 EIGEN_STRONG_INLINE PacketType packet(
Index idx)
const 883 return packet<LoadMode,PacketType>(int(StorageOrder)==
ColMajor?idx:0,int(StorageOrder)==
ColMajor?0:idx);
897 template<
typename ExpressionType,
int S
ide,
bool Transposed,
typename ExpressionShape>
898 struct permutation_matrix_product;
900 template<
typename ExpressionType,
int S
ide,
bool Transposed>
901 struct permutation_matrix_product<ExpressionType, Side, Transposed, DenseShape>
903 typedef typename nested_eval<ExpressionType, 1>::type MatrixType;
904 typedef typename remove_all<MatrixType>::type MatrixTypeCleaned;
906 template<
typename Dest,
typename PermutationType>
907 static inline void run(Dest& dst,
const PermutationType& perm,
const ExpressionType& xpr)
914 if(is_same_dense(dst, mat))
917 Matrix<bool,PermutationType::RowsAtCompileTime,1,0,PermutationType::MaxRowsAtCompileTime> mask(perm.size());
920 while(r < perm.size())
923 while(r<perm.size() && mask[r]) r++;
929 mask.coeffRef(k0) =
true;
930 for(
Index k=perm.indices().coeff(k0); k!=k0; k=perm.indices().coeff(k))
932 Block<Dest, Side==OnTheLeft ? 1 : Dest::RowsAtCompileTime, Side==OnTheRight ? 1 : Dest::ColsAtCompileTime>(dst, k)
933 .swap(Block<Dest, Side==OnTheLeft ? 1 : Dest::RowsAtCompileTime, Side==OnTheRight ? 1 : Dest::ColsAtCompileTime>
934 (dst,((Side==
OnTheLeft) ^ Transposed) ? k0 : kPrev));
936 mask.coeffRef(k) =
true;
943 for(
Index i = 0; i < n; ++i)
945 Block<Dest, Side==OnTheLeft ? 1 : Dest::RowsAtCompileTime, Side==OnTheRight ? 1 : Dest::ColsAtCompileTime>
946 (dst, ((Side==
OnTheLeft) ^ Transposed) ? perm.indices().coeff(i) : i)
950 Block<const MatrixTypeCleaned,Side==OnTheLeft ? 1 : MatrixTypeCleaned::RowsAtCompileTime,Side==OnTheRight ? 1 : MatrixTypeCleaned::ColsAtCompileTime>
951 (mat, ((Side==
OnTheRight) ^ Transposed) ? perm.indices().coeff(i) : i);
957 template<
typename Lhs,
typename Rhs,
int ProductTag,
typename MatrixShape>
958 struct generic_product_impl<Lhs, Rhs, PermutationShape, MatrixShape, ProductTag>
960 template<
typename Dest>
961 static void evalTo(Dest& dst,
const Lhs& lhs,
const Rhs& rhs)
963 permutation_matrix_product<Rhs, OnTheLeft, false, MatrixShape>::run(dst, lhs, rhs);
967 template<
typename Lhs,
typename Rhs,
int ProductTag,
typename MatrixShape>
968 struct generic_product_impl<Lhs, Rhs, MatrixShape, PermutationShape, ProductTag>
970 template<
typename Dest>
971 static void evalTo(Dest& dst,
const Lhs& lhs,
const Rhs& rhs)
973 permutation_matrix_product<Lhs, OnTheRight, false, MatrixShape>::run(dst, rhs, lhs);
977 template<
typename Lhs,
typename Rhs,
int ProductTag,
typename MatrixShape>
978 struct generic_product_impl<Inverse<Lhs>, Rhs, PermutationShape, MatrixShape, ProductTag>
980 template<
typename Dest>
981 static void evalTo(Dest& dst,
const Inverse<Lhs>& lhs,
const Rhs& rhs)
983 permutation_matrix_product<Rhs, OnTheLeft, true, MatrixShape>::run(dst, lhs.nestedExpression(), rhs);
987 template<
typename Lhs,
typename Rhs,
int ProductTag,
typename MatrixShape>
988 struct generic_product_impl<Lhs, Inverse<Rhs>, MatrixShape, PermutationShape, ProductTag>
990 template<
typename Dest>
991 static void evalTo(Dest& dst,
const Lhs& lhs,
const Inverse<Rhs>& rhs)
993 permutation_matrix_product<Lhs, OnTheRight, true, MatrixShape>::run(dst, rhs.nestedExpression(), lhs);
1008 template<
typename ExpressionType,
int S
ide,
bool Transposed,
typename ExpressionShape>
1009 struct transposition_matrix_product
1011 typedef typename nested_eval<ExpressionType, 1>::type MatrixType;
1012 typedef typename remove_all<MatrixType>::type MatrixTypeCleaned;
1014 template<
typename Dest,
typename TranspositionType>
1015 static inline void run(Dest& dst,
const TranspositionType& tr,
const ExpressionType& xpr)
1017 MatrixType mat(xpr);
1018 typedef typename TranspositionType::StorageIndex StorageIndex;
1019 const Index size = tr.size();
1022 if(!is_same_dense(dst,mat))
1025 for(
Index k=(Transposed?size-1:0) ; Transposed?k>=0:k<size ; Transposed?--k:++k)
1026 if(
Index(j=tr.coeff(k))!=k)
1028 if(Side==
OnTheLeft) dst.row(k).swap(dst.row(j));
1029 else if(Side==
OnTheRight) dst.col(k).swap(dst.col(j));
1034 template<
typename Lhs,
typename Rhs,
int ProductTag,
typename MatrixShape>
1035 struct generic_product_impl<Lhs, Rhs, TranspositionsShape, MatrixShape, ProductTag>
1037 template<
typename Dest>
1038 static void evalTo(Dest& dst,
const Lhs& lhs,
const Rhs& rhs)
1040 transposition_matrix_product<Rhs, OnTheLeft, false, MatrixShape>::run(dst, lhs, rhs);
1044 template<
typename Lhs,
typename Rhs,
int ProductTag,
typename MatrixShape>
1045 struct generic_product_impl<Lhs, Rhs, MatrixShape, TranspositionsShape, ProductTag>
1047 template<
typename Dest>
1048 static void evalTo(Dest& dst,
const Lhs& lhs,
const Rhs& rhs)
1050 transposition_matrix_product<Lhs, OnTheRight, false, MatrixShape>::run(dst, rhs, lhs);
1055 template<
typename Lhs,
typename Rhs,
int ProductTag,
typename MatrixShape>
1056 struct generic_product_impl<Transpose<Lhs>, Rhs, TranspositionsShape, MatrixShape, ProductTag>
1058 template<
typename Dest>
1059 static void evalTo(Dest& dst,
const Transpose<Lhs>& lhs,
const Rhs& rhs)
1061 transposition_matrix_product<Rhs, OnTheLeft, true, MatrixShape>::run(dst, lhs.nestedExpression(), rhs);
1065 template<
typename Lhs,
typename Rhs,
int ProductTag,
typename MatrixShape>
1066 struct generic_product_impl<Lhs, Transpose<Rhs>, MatrixShape, TranspositionsShape, ProductTag>
1068 template<
typename Dest>
1069 static void evalTo(Dest& dst,
const Lhs& lhs,
const Transpose<Rhs>& rhs)
1071 transposition_matrix_product<Lhs, OnTheRight, true, MatrixShape>::run(dst, rhs.nestedExpression(), lhs);
1079 #endif // EIGEN_PRODUCT_EVALUATORS_H Definition: Constants.h:320
const int HugeCost
Definition: Constants.h:39
Definition: Constants.h:335
Definition: Constants.h:230
Namespace containing all symbols from the Eigen library.
Definition: Core:271
const unsigned int RowMajorBit
Definition: Constants.h:61
const unsigned int PacketAccessBit
Definition: Constants.h:89
EIGEN_DEFAULT_DENSE_INDEX_TYPE Index
The Index type as used for the API.
Definition: XprHelper.h:35
Definition: Constants.h:333
Definition: Eigen_Colamd.h:50
Definition: Constants.h:322
const int Dynamic
Definition: Constants.h:21
const unsigned int EvalBeforeNestingBit
Definition: Constants.h:65
const unsigned int ActualPacketAccessBit
Definition: Constants.h:100
const unsigned int LinearAccessBit
Definition: Constants.h:125