CUTLASS: default_mma_core_simt.h Source File

| CUTLASS CUDA Templates for Linear Algebra Subroutines and Solvers

- Main Page
- Modules
- Namespaces
- Classes
- Files

- File List
- File Members

default_mma_core_simt.h
Go to the documentation of this file.
1 /***************************************************************************************************
2 * Copyright (c) 2017-2019, NVIDIA CORPORATION. All rights reserved.
3 *
4 * Redistribution and use in source and binary forms, with or without modification, are permitted
5 * provided that the following conditions are met:
6 * * Redistributions of source code must retain the above copyright notice, this list of
7 * conditions and the following disclaimer.
8 * * Redistributions in binary form must reproduce the above copyright notice, this list of
9 * conditions and the following disclaimer in the documentation and/or other materials
10 * provided with the distribution.
11 * * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
12 * to endorse or promote products derived from this software without specific prior written
13 * permission.
14 *
15 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
16 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
17 * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
19 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
20 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
21 * STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
22 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
23 *
24 **************************************************************************************************/
32 #pragma once
33
34 #include "cutlass/cutlass.h"
35 #include "cutlass/array.h"
36 #include "cutlass/fast_math.h"
37
38 #include "cutlass/numeric_types.h"
39 #include "cutlass/matrix_shape.h"
40
41
42 #include "cutlass/transform/pitch_linear_thread_map.h"
43 #include "cutlass/transform/threadblock/regular_tile_iterator_pitch_linear.h"
44 #include "cutlass/transform/threadblock/regular_tile_iterator_pitch_linear_2dthreadtile.h"
45
46 #include "cutlass/gemm/warp/mma_simt_policy.h"
47 #include "cutlass/gemm/warp/mma_simt.h"
48 #include "cutlass/gemm/threadblock/default_mma_core.h"
49
51
52 namespace cutlass {
53 namespace gemm {
54 namespace threadblock {
55
56 namespace detail {
57
58 // convert a WarpShape which is the whole tile of elements into warp num threads.
59 // The goal is for each thread's tile of elements to be as square as possible
60 // for performance (4x4 will be faster than 2x8).
61 template<typename WarpShape>
62 constexpr int simt_get_warp_threads_m() {
63 return (WarpShape::kM > WarpShape::kN) ? 8 : 4;
64 }
65
67 constexpr int simt_transpose_padding(int threads, int crosswise, int size_in_bits) {
68 return (size_in_bits >= 32 ?
69 threads / crosswise / (size_in_bits / 32) :
70 threads / crosswise * (32 / size_in_bits)
71 );
72 }
73
74 }
75
77
85 template <
88 typename Shape_,
90 typename WarpShape_,
92 typename ElementA_,
94 typename ElementB_,
96 typename ElementC_,
98 typename LayoutC_,
100 typename Operator_>
101 struct DefaultMmaCore<Shape_, WarpShape_, GemmShape<1, 1, 1>, ElementA_,
102 layout::ColumnMajor, ElementB_, layout::RowMajor,
103 ElementC_, LayoutC_, arch::OpClassSimt, 2, Operator_
104 > {
105 using Shape = Shape_;
106 using WarpShape = WarpShape_;
107 using InstructionShape = GemmShape<1, 1, 1>;
108 using ElementA = ElementA_;
109 using LayoutA = layout::ColumnMajor;
110 using ElementB = ElementB_;
111 using LayoutB = layout::RowMajor;
112 using ElementC = ElementC_;
113 using LayoutC = LayoutC_;
114 using OperatorClass = arch::OpClassSimt;
115 static int const PartitionsK = Shape::kK / WarpShape::kK;
116
118 using Operator = Operator_;
119
121 using WarpCount = GemmShape<
122 Shape::kM / WarpShape::kM,
123 Shape::kN / WarpShape::kN,
124 PartitionsK
125 >;
126
127 // Divisility requirements
128 static_assert(
129 !(Shape::kM % WarpShape::kM) &&
130 !(Shape::kN % WarpShape::kN),
131 "Threadblock-scoped GEMM should be divisible by warp-scoped GEMM size."
132 );
133
135 static int const kWarpSize = warp::WarpSize<arch::OpClassSimt>::value;
136
138 static int const kThreads = WarpCount::kCount * kWarpSize;
139
140 static int const kElementsPerAccess = 1;
141
142 //
143 // Shared memory layouts
144 //
145
146 using SmemLayoutA = layout::ColumnMajor;
147 using SmemLayoutB = layout::RowMajor;
148
149 //
150 // Iterators to write to shared memory
151 //
152
154 using IteratorThreadMapA = transform::PitchLinearStripminedThreadMap<
155 layout::PitchLinearShape<Shape::kM, Shape::kK>,
156 kThreads,
157 kElementsPerAccess
158 >;
159
161 using SmemIteratorA = transform::threadblock::RegularTileIterator<
162 MatrixShape<Shape::kM, Shape::kK>,
163 ElementA,
164 SmemLayoutA,
165 1,
166 IteratorThreadMapA
167 >;
168
170 using IteratorThreadMapB = transform::PitchLinearStripminedThreadMap<
171 layout::PitchLinearShape<Shape::kN, Shape::kK>,
172 kThreads,
173 kElementsPerAccess
174 >;
175
177 using SmemIteratorB = transform::threadblock::RegularTileIterator<
178 MatrixShape<Shape::kK, Shape::kN>,
179 ElementB,
180 SmemLayoutB,
181 0,
182 IteratorThreadMapB
183 >;
184
185 //
186 // Warp-level matrix multiply operator
187 //
188
189 // Define the warp-level op
190 static const int WarpNumThreadsM = detail::simt_get_warp_threads_m<WarpShape>();
191 static const int WarpNumThreadsN = kWarpSize / WarpNumThreadsM;
192 static const int ThreadTileM = WarpShape::kM / WarpNumThreadsM;
193 static const int ThreadTileN = WarpShape::kN / WarpNumThreadsN;
194 static_assert(!(WarpShape::kM % WarpNumThreadsM) && !(WarpShape::kN % WarpNumThreadsN),
195 "WarpShape must be divisible by ThreadTile shape.");
196 static const int LaneLayout = ThreadTileM > 4 && ThreadTileN > 4 ? 2 : 1;
197 static const int numElementsA = 128 / sizeof_bits<ElementA>::value;
198 static const int numElementsB = 128 / sizeof_bits<ElementB>::value;
199 static const int LaneM = cutlass::const_min(numElementsA, ThreadTileM);
200 static const int LaneN = cutlass::const_min(numElementsB, ThreadTileN);
201 // these should have max of thread tile also
202 using LaneMmaShape = cutlass::gemm::GemmShape<
203 LaneM,
204 LaneN,
205 1>;
206 using Policy = cutlass::gemm::warp::MmaSimtPolicy<
207 cutlass::MatrixShape<WarpNumThreadsM, WarpNumThreadsN>, // WarpShape
208 cutlass::layout::RowMajorInterleaved<LaneLayout>, // LaneLayout
209 LaneMmaShape
210 >;
211
212 using MmaWarpSimt = cutlass::gemm::warp::MmaSimt<
213 WarpShape,
214 ElementA,
215 SmemLayoutA,
216 ElementB,
217 SmemLayoutB,
218 ElementC,
219 LayoutC,
220 Policy
221 >;
222
224 using MmaPolicy = MmaPolicy<
225 MmaWarpSimt,
226 MatrixShape<0, 0>,
227 MatrixShape<0, 0>,
228 WarpCount::kK
229 >;
230 };
231
233
241 template <
244 typename Shape_,
246 typename WarpShape_,
248 typename ElementA_,
250 typename ElementB_,
252 typename ElementC_,
254 typename LayoutC_,
256 typename Operator_>
257 struct DefaultMmaCore<Shape_, WarpShape_, GemmShape<1, 1, 1>, ElementA_,
258 layout::RowMajor, ElementB_, layout::ColumnMajor,
259 ElementC_, LayoutC_, arch::OpClassSimt, 2, Operator_
260 > {
261 using Shape = Shape_;
262 using WarpShape = WarpShape_;
263 using InstructionShape = GemmShape<1, 1, 1>;
264 using ElementA = ElementA_;
265 using LayoutA = layout::RowMajor;
266 using ElementB = ElementB_;
267 using LayoutB = layout::ColumnMajor;
268 using ElementC = ElementC_;
269 using LayoutC = LayoutC_;
270 using OperatorClass = arch::OpClassSimt;
271 static int const PartitionsK = Shape::kK / WarpShape::kK;
272
274 using Operator = Operator_;
275
277 using WarpCount = GemmShape<
278 Shape::kM / WarpShape::kM,
279 Shape::kN / WarpShape::kN,
280 PartitionsK
281 >;
282
283 // Divisility requirements
284 static_assert(
285 !(Shape::kM % WarpShape::kM) &&
286 !(Shape::kN % WarpShape::kN),
287 "Threadblock-scoped GEMM should be divisible by warp-scoped GEMM size."
288 );
289
291 static int const kWarpSize = warp::WarpSize<arch::OpClassSimt>::value;
292
294 static int const kThreads = WarpCount::kCount * kWarpSize;
295
296 static int const kElementsPerAccess = 1;
297
298 //
299 // Shared memory layouts
300 //
301
302 using SmemLayoutA = layout::ColumnMajor;
303 using SmemLayoutB = layout::RowMajor;
304
305 //
306 // Iterators to write to shared memory
307 //
308
310 using IteratorThreadMapA = transform::PitchLinearStripminedThreadMap<
311 layout::PitchLinearShape<Shape::kK, Shape::kM>,
312 kThreads,
313 kElementsPerAccess
314 >;
315
317 using SmemThreadMapA = transform::TransposePitchLinearThreadMapSimt<IteratorThreadMapA>;
318
320 using SmemIteratorA = transform::threadblock::RegularTileIterator<
321 MatrixShape<Shape::kM, Shape::kK>,
322 ElementA,
323 SmemLayoutA,
324 1,
325 SmemThreadMapA // was IteratorThreadMapA
326 >;
327
329 using IteratorThreadMapB = transform::PitchLinearStripminedThreadMap<
330 layout::PitchLinearShape<Shape::kK, Shape::kN>,
331 kThreads,
332 kElementsPerAccess
333 >;
334
336 using SmemThreadMapB = transform::TransposePitchLinearThreadMapSimt<IteratorThreadMapB>;
337
339 using SmemIteratorB = transform::threadblock::RegularTileIterator<
340 MatrixShape<Shape::kK, Shape::kN>,
341 ElementB,
342 SmemLayoutB,
343 0,
344 SmemThreadMapB // was IteratorThreadMapA
345 >;
346
347 //
348 // Warp-level matrix multiply operator
349 //
350
351 // Define the warp-level op
352 static const int WarpNumThreadsM = detail::simt_get_warp_threads_m<WarpShape>();
353 static const int WarpNumThreadsN = kWarpSize / WarpNumThreadsM;
354 static const int ThreadTileM = WarpShape::kM / WarpNumThreadsM;
355 static const int ThreadTileN = WarpShape::kN / WarpNumThreadsN;
356 static_assert(!(WarpShape::kM % WarpNumThreadsM) && !(WarpShape::kN % WarpNumThreadsN),
357 "WarpShape must be divisible by ThreadTile shape.");
358 static const int LaneLayout = ThreadTileM > 4 && ThreadTileN > 4 ? 2 : 1;
359 static const int numElementsA = 128 / sizeof_bits<ElementA>::value;
360 static const int numElementsB = 128 / sizeof_bits<ElementB>::value;
361 static const int LaneM = cutlass::const_min(numElementsA, ThreadTileM);
362 static const int LaneN = cutlass::const_min(numElementsB, ThreadTileN);
363
364 static int const kPaddingM = detail::simt_transpose_padding(kWarpSize, Shape::kK, sizeof_bits<ElementA>::value);
365 static int const kPaddingN = detail::simt_transpose_padding(kWarpSize, Shape::kK, sizeof_bits<ElementB>::value);
366
367 // these should have max of thread tile also
368 using LaneMmaShape = cutlass::gemm::GemmShape<
369 LaneM,
370 LaneN,
371 1>;
372 using Policy = cutlass::gemm::warp::MmaSimtPolicy<
373 cutlass::MatrixShape<WarpNumThreadsM, WarpNumThreadsN>, // WarpShape
374 cutlass::layout::RowMajorInterleaved<LaneLayout>, // LaneLayout
375 LaneMmaShape
376 >;
377
378 using MmaWarpSimt = cutlass::gemm::warp::MmaSimt<
379 WarpShape,
380 ElementA,
381 SmemLayoutA,
382 ElementB,
383 SmemLayoutB,
384 ElementC,
385 LayoutC,
386 Policy
387 >;
388
390 using MmaPolicy = MmaPolicy<
391 MmaWarpSimt,
392 MatrixShape<kPaddingN, 0>, // skew for A matrix to avoid SMEM bank conflicts
393 MatrixShape<0, kPaddingN>, // skew for B matrix to avoid SMEM bank conflicts
394 WarpCount::kK
395 >;
396 };
397
399
407 template <
410 typename Shape_,
412 typename WarpShape_,
414 typename ElementA_,
416 typename ElementB_,
418 typename ElementC_,
420 typename LayoutC_,
422 typename Operator_>
423 struct DefaultMmaCore<Shape_, WarpShape_, GemmShape<1, 1, 1>, ElementA_,
424 layout::RowMajor, ElementB_, layout::RowMajor, ElementC_,
425 LayoutC_, arch::OpClassSimt, 2, Operator_
426 > {
427 using Shape = Shape_;
428 using WarpShape = WarpShape_;
429 using InstructionShape = GemmShape<1, 1, 1>;
430 using ElementA = ElementA_;
431 using LayoutA = layout::RowMajor;
432 using ElementB = ElementB_;
433 using LayoutB = layout::RowMajor;
434 using ElementC = ElementC_;
435 using LayoutC = LayoutC_;
436 using OperatorClass = arch::OpClassSimt;
437 static int const PartitionsK = Shape::kK / WarpShape::kK;
438
440 using Operator = Operator_;
441
443 using WarpCount = GemmShape<
444 Shape::kM / WarpShape::kM,
445 Shape::kN / WarpShape::kN,
446 PartitionsK
447 >;
448
449 // Divisility requirements
450 static_assert(
451 !(Shape::kM % WarpShape::kM) &&
452 !(Shape::kN % WarpShape::kN),
453 "Threadblock-scoped GEMM should be divisible by warp-scoped GEMM size."
454 );
455
457 static int const kWarpSize = warp::WarpSize<arch::OpClassSimt>::value;
458
460 static int const kThreads = WarpCount::kCount * kWarpSize;
461
462 static int const kElementsPerAccess = 1;
463
464 //
465 // Shared memory layouts
466 //
467
468 using SmemLayoutA = layout::ColumnMajor;
469 using SmemLayoutB = layout::RowMajor;
470
471 //
472 // Iterators to write to shared memory
473 //
474
476 using IteratorThreadMapA = transform::PitchLinearStripminedThreadMap<
477 layout::PitchLinearShape<Shape::kK, Shape::kM>,
478 kThreads,
479 kElementsPerAccess
480 >;
481
483 using SmemThreadMapA = transform::TransposePitchLinearThreadMapSimt<IteratorThreadMapA>;
484
486 using SmemIteratorA = transform::threadblock::RegularTileIterator<
487 MatrixShape<Shape::kM, Shape::kK>,
488 ElementA,
489 SmemLayoutA,
490 1,
491 SmemThreadMapA
492 >;
493
495 using IteratorThreadMapB = transform::PitchLinearStripminedThreadMap<
496 layout::PitchLinearShape<Shape::kN, Shape::kK>,
497 kThreads,
498 kElementsPerAccess
499 >;
500
502 using SmemIteratorB = transform::threadblock::RegularTileIterator<
503 MatrixShape<Shape::kK, Shape::kN>,
504 ElementB,
505 SmemLayoutB,
506 0,
507 IteratorThreadMapB
508 >;
509
510 //
511 // Warp-level matrix multiply operator
512 //
513
514 // Define the warp-level op
515 static const int WarpNumThreadsM = detail::simt_get_warp_threads_m<WarpShape>();
516 static const int WarpNumThreadsN = kWarpSize / WarpNumThreadsM;
517 static const int ThreadTileM = WarpShape::kM / WarpNumThreadsM;
518 static const int ThreadTileN = WarpShape::kN / WarpNumThreadsN;
519 static_assert(!(WarpShape::kM % WarpNumThreadsM) && !(WarpShape::kN % WarpNumThreadsN),
520 "WarpShape must be divisible by ThreadTile shape.");
521 static const int LaneLayout = ThreadTileM > 4 && ThreadTileN > 4 ? 2 : 1;
522 static const int numElementsA = 128 / sizeof_bits<ElementA>::value;
523 static const int numElementsB = 128 / sizeof_bits<ElementB>::value;
524 static const int LaneM = cutlass::const_min(numElementsA, ThreadTileM);
525 static const int LaneN = cutlass::const_min(numElementsB, ThreadTileN);
526
527 static int const kPaddingM = detail::simt_transpose_padding(kWarpSize, Shape::kK, sizeof_bits<ElementA>::value);
528
529 // these should have max of thread tile also
530 using LaneMmaShape = cutlass::gemm::GemmShape<
531 LaneM,
532 LaneN,
533 1>;
534 using Policy = cutlass::gemm::warp::MmaSimtPolicy<
535 cutlass::MatrixShape<WarpNumThreadsM, WarpNumThreadsN>, // WarpShape
536 cutlass::layout::RowMajorInterleaved<LaneLayout>, // LaneLayout
537 LaneMmaShape
538 >;
539
540 using MmaWarpSimt = cutlass::gemm::warp::MmaSimt<
541 WarpShape,
542 ElementA,
543 SmemLayoutA,
544 ElementB,
545 SmemLayoutB,
546 ElementC,
547 LayoutC,
548 Policy
549 >;
550
552 using MmaPolicy = MmaPolicy<
553 MmaWarpSimt,
554 MatrixShape<kPaddingM, 0>, // skew for A matrix to avoid SMEM bank conflicts
555 MatrixShape<0, 0>,
556 WarpCount::kK
557 >;
558 };
559
561
569 template <
572 typename Shape_,
574 typename WarpShape_,
576 typename ElementA_,
578 typename ElementB_,
580 typename ElementC_,
582 typename LayoutC_,
584 typename Operator_>
585 struct DefaultMmaCore<Shape_, WarpShape_, GemmShape<1, 1, 1>, ElementA_,
586 layout::ColumnMajor, ElementB_, layout::ColumnMajor,
587 ElementC_, LayoutC_, arch::OpClassSimt, 2, Operator_
588 > {
589 using Shape = Shape_;
590 using WarpShape = WarpShape_;
591 using InstructionShape = GemmShape<1, 1, 1>;
592 using ElementA = ElementA_;
593 using LayoutA = layout::ColumnMajor;
594 using ElementB = ElementB_;
595 using LayoutB = layout::ColumnMajor;
596 using ElementC = ElementC_;
597 using LayoutC = LayoutC_;
598 using OperatorClass = arch::OpClassSimt;
599 static int const PartitionsK = Shape::kK / WarpShape::kK;
600
602 using Operator = Operator_;
603
605 using WarpCount = GemmShape<
606 Shape::kM / WarpShape::kM,
607 Shape::kN / WarpShape::kN,
608 PartitionsK
609 >;
610
611 // Divisility requirements
612 static_assert(
613 !(Shape::kM % WarpShape::kM) &&
614 !(Shape::kN % WarpShape::kN),
615 "Threadblock-scoped GEMM should be divisible by warp-scoped GEMM size."
616 );
617
619 static int const kWarpSize = warp::WarpSize<arch::OpClassSimt>::value;
620
622 static int const kThreads = WarpCount::kCount * kWarpSize;
623
624 static int const kElementsPerAccess = 1;
625
626 //
627 // Shared memory layouts
628 //
629
630 using SmemLayoutA = layout::ColumnMajor;
631 using SmemLayoutB = layout::RowMajor;
632
633 //
634 // Iterators to write to shared memory
635 //
636
638 using IteratorThreadMapA = transform::PitchLinearStripminedThreadMap<
639 layout::PitchLinearShape<Shape::kM, Shape::kK>,
640 kThreads,
641 kElementsPerAccess
642 >;
643
645 using SmemIteratorA = transform::threadblock::RegularTileIterator<
646 MatrixShape<Shape::kM, Shape::kK>,
647 ElementA,
648 SmemLayoutA,
649 1,
650 IteratorThreadMapA
651 >;
652
654 using IteratorThreadMapB = transform::PitchLinearStripminedThreadMap<
655 layout::PitchLinearShape<Shape::kK, Shape::kN>,
656 kThreads,
657 kElementsPerAccess
658 >;
659
661 using SmemThreadMapB = transform::TransposePitchLinearThreadMapSimt<IteratorThreadMapB>;
662
664 using SmemIteratorB = transform::threadblock::RegularTileIterator<
665 MatrixShape<Shape::kK, Shape::kN>,
666 ElementB,
667 SmemLayoutB,
668 0,
669 SmemThreadMapB
670 >;
671
672 //
673 // Warp-level matrix multiply operator
674 //
675
676 // Define the warp-level op
677 static const int WarpNumThreadsM = detail::simt_get_warp_threads_m<WarpShape>();
678 static const int WarpNumThreadsN = kWarpSize / WarpNumThreadsM;
679 static const int ThreadTileM = WarpShape::kM / WarpNumThreadsM;
680 static const int ThreadTileN = WarpShape::kN / WarpNumThreadsN;
681 static_assert(!(WarpShape::kM % WarpNumThreadsM) && !(WarpShape::kN % WarpNumThreadsN),
682 "WarpShape must be divisible by ThreadTile shape.");
683 static const int LaneLayout = ThreadTileM > 4 && ThreadTileN > 4 ? 2 : 1;
684 static const int numElementsA = 128 / sizeof_bits<ElementA>::value;
685 static const int numElementsB = 128 / sizeof_bits<ElementB>::value;
686 static const int LaneM = cutlass::const_min(numElementsA, ThreadTileM);
687 static const int LaneN = cutlass::const_min(numElementsB, ThreadTileN);
688
689 static int const kPaddingN = detail::simt_transpose_padding(kWarpSize, Shape::kK, sizeof_bits<ElementB>::value);
690
691 // these should have max of thread tile also
692 using LaneMmaShape = cutlass::gemm::GemmShape<
693 LaneM,
694 LaneN,
695 1>;
696 using Policy = cutlass::gemm::warp::MmaSimtPolicy<
697 cutlass::MatrixShape<WarpNumThreadsM, WarpNumThreadsN>, // WarpShape
698 cutlass::layout::RowMajorInterleaved<LaneLayout>, // LaneLayout
699 LaneMmaShape
700 >;
701
702 using MmaWarpSimt = cutlass::gemm::warp::MmaSimt<
703 WarpShape,
704 ElementA,
705 SmemLayoutA,
706 ElementB,
707 SmemLayoutB,
708 ElementC,
709 LayoutC,
710 Policy
711 >;
712
714 using MmaPolicy = MmaPolicy<
715 MmaWarpSimt,
716 MatrixShape<0, 0>,
717 MatrixShape<0, kPaddingN>, // skew for B matrix to avoid SMEM bank conflicts
718 WarpCount::kK
719 >;
720 };
721
723
731 template <
734 typename Shape_,
736 typename WarpShape_,
738 typename ElementC_,
740 typename LayoutC_,
742 typename Operator_>
743 struct DefaultMmaCore<Shape_, WarpShape_, GemmShape<1, 1, 4>, int8_t,
744 layout::ColumnMajor, int8_t, layout::RowMajor, ElementC_,
745 LayoutC_, arch::OpClassSimt, 2, Operator_
746 > {
747
748 using Shape = Shape_;
749 using WarpShape = WarpShape_;
750 using InstructionShape = GemmShape<1, 1, 4>;
751 using ElementA = int8_t;
752 using LayoutA = layout::ColumnMajor;
753 using ElementB = int8_t;
754 using LayoutB = layout::RowMajor;
755 using ElementC = ElementC_;
756 using LayoutC = LayoutC_;
757 using OperatorClass = arch::OpClassSimt;
758 static int const PartitionsK = Shape::kK / WarpShape::kK;
759
761 using Operator = Operator_;
762
764 using WarpCount = GemmShape<
765 Shape::kM / WarpShape::kM,
766 Shape::kN / WarpShape::kN,
767 PartitionsK
768 >;
769
770 // Divisility requirements
771 static_assert(
772 !(Shape::kM % WarpShape::kM) &&
773 !(Shape::kN % WarpShape::kN),
774 "Threadblock-scoped GEMM should be divisible by warp-scoped GEMM size."
775 );
776
778 static int const kWarpSize = warp::WarpSize<arch::OpClassSimt>::value;
779
781 static int const kThreads = WarpCount::kCount * kWarpSize;
782
783 //
784 // Shared memory layouts
785 //
786
787 using SmemLayoutA = layout::ColumnMajorInterleaved<4>;
788 using SmemLayoutB = layout::RowMajorInterleaved<4>;
789
790 //
791 // Iterators to write to shared memory
792 //
793
795 using IteratorThreadMapA = transform::PitchLinear2DThreadTileStripminedThreadMap<
796 layout::PitchLinearShape<Shape::kM, Shape::kK>,
797 kThreads,
798 layout::PitchLinearShape<4, 4>
799 >;
800
802 using SmemIteratorA = transform::threadblock::RegularTileIterator2dThreadTile<
803 MatrixShape<Shape::kM, Shape::kK>,
804 ElementA,
805 SmemLayoutA,
806 1,
807 IteratorThreadMapA
808 >;
809
810
812 using IteratorThreadMapB = transform::PitchLinear2DThreadTileStripminedThreadMap<
813 layout::PitchLinearShape<Shape::kN, Shape::kK>,
814 kThreads,
815 layout::PitchLinearShape<4, 4>
816 >;
817
819 using SmemIteratorB = transform::threadblock::RegularTileIterator2dThreadTile<
820 MatrixShape<Shape::kK, Shape::kN>,
821 ElementB,
822 SmemLayoutB,
823 0,
824 IteratorThreadMapB
825 >;
826
827 //
828 // Warp-level matrix multiply operator
829 //
830
831 // Define the warp-level op
832 static const int WarpNumThreadsM = detail::simt_get_warp_threads_m<WarpShape>();
833 static const int WarpNumThreadsN = kWarpSize / WarpNumThreadsM;
834 static const int ThreadTileM = WarpShape::kM / WarpNumThreadsM;
835 static const int ThreadTileN = WarpShape::kN / WarpNumThreadsN;
836 static_assert(!(WarpShape::kM % WarpNumThreadsM) && !(WarpShape::kN % WarpNumThreadsN),
837 "WarpShape must be divisible by ThreadTile shape.");
838 static const int LaneLayout = ThreadTileM > 4 && ThreadTileN > 4 ? 2 : 1;
839 static const int numElementsA = 128 / sizeof_bits<ElementA>::value;
840 static const int numElementsB = 128 / sizeof_bits<ElementB>::value;
841 static const int LaneM = cutlass::const_min(4, ThreadTileM);
842 static const int LaneN = cutlass::const_min(4, ThreadTileN);
843 // these should have max of thread tile also
844 using LaneMmaShape = cutlass::gemm::GemmShape<
845 LaneM,
846 LaneN,
847 4>;
848
849 using Policy = cutlass::gemm::warp::MmaSimtPolicy<
850 cutlass::MatrixShape<WarpNumThreadsM, WarpNumThreadsN>, // WarpShape
851 cutlass::layout::ColumnMajorInterleaved<LaneLayout>, // LaneLayout
852 LaneMmaShape
853 >;
854
855 using MmaWarpSimt = cutlass::gemm::warp::MmaSimt<
856 WarpShape,
857 ElementA,
858 SmemLayoutA,
859 ElementB,
860 SmemLayoutB,
861 ElementC,
862 LayoutC,
863 Policy,
864 PartitionsK
865 >;
866
868 using MmaPolicy = MmaPolicy<
869 MmaWarpSimt,
870 MatrixShape<0, 0>,
871 MatrixShape<0, 0>,
872 WarpCount::kK
873 >;
874 };
875
878 //
885 template <
888 typename Shape_,
890 typename WarpShape_,
892 typename ElementC_,
894 typename LayoutC_,
896 typename Operator_>
897 struct DefaultMmaCore<Shape_, WarpShape_, GemmShape<1, 1, 4>, int8_t,
898 layout::RowMajor, int8_t, layout::ColumnMajor, ElementC_,
899 LayoutC_, arch::OpClassSimt, 2, Operator_
900 > {
901
902 using Shape = Shape_;
903 using WarpShape = WarpShape_;
904 using InstructionShape = GemmShape<1, 1, 4>;
905 using ElementA = int8_t;
906 using LayoutA = layout::RowMajor;
907 using ElementB = int8_t;
908 using LayoutB = layout::ColumnMajor;
909 using ElementC = ElementC_;
910 using LayoutC = LayoutC_;
911 using OperatorClass = arch::OpClassSimt;
912 static int const PartitionsK = Shape::kK / WarpShape::kK;
913
915 using Operator = Operator_;
916
918 using WarpCount = GemmShape<
919 Shape::kM / WarpShape::kM,
920 Shape::kN / WarpShape::kN,
921 PartitionsK
922 >;
923
924 // Divisility requirements
925 static_assert(
926 !(Shape::kM % WarpShape::kM) &&
927 !(Shape::kN % WarpShape::kN),
928 "Threadblock-scoped GEMM should be divisible by warp-scoped GEMM size."
929 );
930
932 static int const kWarpSize = warp::WarpSize<arch::OpClassSimt>::value;
933
935 static int const kThreads = WarpCount::kCount * kWarpSize;
936
937 //
938 // Shared memory layouts
939 //
940
941 using SmemLayoutA = layout::ColumnMajorInterleaved<4>;
942 using SmemLayoutB = layout::RowMajorInterleaved<4>;
943
944 //
945 // Iterators to write to shared memory
946 //
947
949 using IteratorThreadMapA = transform::PitchLinear2DThreadTileStripminedThreadMap<
950 layout::PitchLinearShape<Shape::kK, Shape::kM>,
951 kThreads,
952 layout::PitchLinearShape<4, 4>
953 >;
954
956 using SmemThreadMapA = transform::TransposePitchLinearThreadMap2DThreadTile<IteratorThreadMapA>;
957
959 using SmemIteratorA = transform::threadblock::RegularTileIterator2dThreadTile<
960 MatrixShape<Shape::kM, Shape::kK>,
961 ElementA,
962 SmemLayoutA,
963 1,
964 SmemThreadMapA
965 >;
966
967
969 using IteratorThreadMapB = transform::PitchLinear2DThreadTileStripminedThreadMap<
970 layout::PitchLinearShape<Shape::kK, Shape::kN>,
971 kThreads,
972 layout::PitchLinearShape<4, 4>
973 >;
974
976 using SmemThreadMapB = transform::TransposePitchLinearThreadMap2DThreadTile<IteratorThreadMapB>;
977
979 using SmemIteratorB = transform::threadblock::RegularTileIterator2dThreadTile<
980 MatrixShape<Shape::kK, Shape::kN>,
981 ElementB,
982 SmemLayoutB,
983 0,
984 SmemThreadMapB
985 >;
986
987 //
988 // Warp-level matrix multiply operator
989 //
990
991 // Define the warp-level op
992 static const int WarpNumThreadsM = detail::simt_get_warp_threads_m<WarpShape>();
993 static const int WarpNumThreadsN = kWarpSize / WarpNumThreadsM;
994 static const int ThreadTileM = WarpShape::kM / WarpNumThreadsM;
995 static const int ThreadTileN = WarpShape::kN / WarpNumThreadsN;
996 static_assert(!(WarpShape::kM % WarpNumThreadsM) && !(WarpShape::kN % WarpNumThreadsN),
997 "WarpShape must be divisible by ThreadTile shape.");
998 static const int LaneLayout = ThreadTileM > 4 && ThreadTileN > 4 ? 2 : 1;
999 static const int numElementsA = 128 / sizeof_bits<ElementA>::value;
1000 static const int numElementsB = 128 / sizeof_bits<ElementB>::value;
1001 static const int LaneM = cutlass::const_min(4, ThreadTileM);
1002 static const int LaneN = cutlass::const_min(4, ThreadTileN);
1003 // these should have max of thread tile also
1004 using LaneMmaShape = cutlass::gemm::GemmShape<
1005 LaneM,
1006 LaneN,
1007 4>;
1008
1009 using Policy = cutlass::gemm::warp::MmaSimtPolicy<
1010 cutlass::MatrixShape<WarpNumThreadsM, WarpNumThreadsN>, // WarpShape
1011 cutlass::layout::ColumnMajorInterleaved<LaneLayout>, // LaneLayout
1012 LaneMmaShape
1013 >;
1014
1015 using MmaWarpSimt = cutlass::gemm::warp::MmaSimt<
1016 WarpShape,
1017 ElementA,
1018 SmemLayoutA,
1019 ElementB,
1020 SmemLayoutB,
1021 ElementC,
1022 LayoutC,
1023 Policy,
1024 PartitionsK
1025 >;
1026
1027 static int const kPaddingM = detail::simt_transpose_padding(kWarpSize, Shape::kK, sizeof_bits<ElementA>::value);
1028 static int const kPaddingN = detail::simt_transpose_padding(kWarpSize, Shape::kK, sizeof_bits<ElementB>::value);
1029
1031 using MmaPolicy = MmaPolicy<
1032 MmaWarpSimt,
1033 MatrixShape<kPaddingM, 0>,
1034 MatrixShape<0, kPaddingN>,
1035 WarpCount::kK
1036 >;
1037 };
1038
1041 //
1048 template <
1051 typename Shape_,
1053 typename WarpShape_,
1055 typename ElementC_,
1057 typename LayoutC_,
1059 typename Operator_>
1060 struct DefaultMmaCore<Shape_, WarpShape_, GemmShape<1, 1, 4>, int8_t,
1061 layout::RowMajor, int8_t, layout::RowMajor, ElementC_,
1062 LayoutC_, arch::OpClassSimt, 2, Operator_
1063 > {
1064
1065 using Shape = Shape_;
1066 using WarpShape = WarpShape_;
1067 using InstructionShape = GemmShape<1, 1, 4>;
1068 using ElementA = int8_t;
1069 using LayoutA = layout::RowMajor;
1070 using ElementB = int8_t;
1071 using LayoutB = layout::RowMajor;
1072 using ElementC = ElementC_;
1073 using LayoutC = LayoutC_;
1074 using OperatorClass = arch::OpClassSimt;
1075 static int const PartitionsK = Shape::kK / WarpShape::kK;
1076
1078 using Operator = Operator_;
1079
1081 using WarpCount = GemmShape<
1082 Shape::kM / WarpShape::kM,
1083 Shape::kN / WarpShape::kN,
1084 PartitionsK
1085 >;
1086
1087 // Divisility requirements
1088 static_assert(
1089 !(Shape::kM % WarpShape::kM) &&
1090 !(Shape::kN % WarpShape::kN),
1091 "Threadblock-scoped GEMM should be divisible by warp-scoped GEMM size."
1092 );
1093
1095 static int const kWarpSize = warp::WarpSize<arch::OpClassSimt>::value;
1096
1098 static int const kThreads = WarpCount::kCount * kWarpSize;
1099
1100 //
1101 // Shared memory layouts
1102 //
1103
1104 using SmemLayoutA = layout::ColumnMajorInterleaved<4>;
1105 using SmemLayoutB = layout::RowMajorInterleaved<4>;
1106
1107 //
1108 // Iterators to write to shared memory
1109 //
1110
1112 using IteratorThreadMapA = transform::PitchLinear2DThreadTileStripminedThreadMap<
1113 layout::PitchLinearShape<Shape::kK, Shape::kM>,
1114 kThreads,
1115 layout::PitchLinearShape<4, 4>
1116 >;
1117
1119 using SmemThreadMapA = transform::TransposePitchLinearThreadMap2DThreadTile<IteratorThreadMapA>;
1120
1122 using SmemIteratorA = transform::threadblock::RegularTileIterator2dThreadTile<
1123 MatrixShape<Shape::kM, Shape::kK>,
1124 ElementA,
1125 SmemLayoutA,
1126 1,
1127 SmemThreadMapA
1128 >;
1129
1131 using IteratorThreadMapB = transform::PitchLinear2DThreadTileStripminedThreadMap<
1132 layout::PitchLinearShape<Shape::kN, Shape::kK>,
1133 kThreads,
1134 layout::PitchLinearShape<4, 4>
1135 >;
1136
1138 using SmemIteratorB = transform::threadblock::RegularTileIterator2dThreadTile<
1139 MatrixShape<Shape::kK, Shape::kN>,
1140 ElementB,
1141 SmemLayoutB,
1142 0,
1143 IteratorThreadMapB
1144 >;
1145
1146 //
1147 // Warp-level matrix multiply operator
1148 //
1149
1150 // Define the warp-level op
1151 static const int WarpNumThreadsM = detail::simt_get_warp_threads_m<WarpShape>();
1152 static const int WarpNumThreadsN = kWarpSize / WarpNumThreadsM;
1153 static const int ThreadTileM = WarpShape::kM / WarpNumThreadsM;
1154 static const int ThreadTileN = WarpShape::kN / WarpNumThreadsN;
1155 static_assert(!(WarpShape::kM % WarpNumThreadsM) && !(WarpShape::kN % WarpNumThreadsN),
1156 "WarpShape must be divisible by ThreadTile shape.");
1157 static const int LaneLayout = ThreadTileM > 4 && ThreadTileN > 4 ? 2 : 1;
1158 static const int numElementsA = 128 / sizeof_bits<ElementA>::value;
1159 static const int numElementsB = 128 / sizeof_bits<ElementB>::value;
1160 static const int LaneM = cutlass::const_min(4, ThreadTileM);
1161 static const int LaneN = cutlass::const_min(4, ThreadTileN);
1162 // these should have max of thread tile also
1163 using LaneMmaShape = cutlass::gemm::GemmShape<
1164 LaneM,
1165 LaneN,
1166 4>;
1167
1168 using Policy = cutlass::gemm::warp::MmaSimtPolicy<
1169 cutlass::MatrixShape<WarpNumThreadsM, WarpNumThreadsN>, // WarpShape
1170 cutlass::layout::ColumnMajorInterleaved<LaneLayout>, // LaneLayout
1171 LaneMmaShape
1172 >;
1173
1174 using MmaWarpSimt = cutlass::gemm::warp::MmaSimt<
1175 WarpShape,
1176 ElementA,
1177 SmemLayoutA,
1178 ElementB,
1179 SmemLayoutB,
1180 ElementC,
1181 LayoutC,
1182 Policy,
1183 PartitionsK
1184 >;
1185
1186 static int const kPaddingM = detail::simt_transpose_padding(kWarpSize, Shape::kK, sizeof_bits<ElementA>::value);
1187 static int const kPaddingN = detail::simt_transpose_padding(kWarpSize, Shape::kK, sizeof_bits<ElementB>::value);
1188
1190 using MmaPolicy = MmaPolicy<
1191 MmaWarpSimt,
1192 MatrixShape<kPaddingM, 0>,
1193 MatrixShape<0, 0>,
1194 WarpCount::kK
1195 >;
1196 };
1197
1200 //
1207 template <
1210 typename Shape_,
1212 typename WarpShape_,
1214 typename ElementC_,
1216 typename LayoutC_,
1218 typename Operator_>
1219 struct DefaultMmaCore<Shape_, WarpShape_, GemmShape<1, 1, 4>, int8_t,
1220 layout::ColumnMajor, int8_t, layout::ColumnMajor, ElementC_,
1221 LayoutC_, arch::OpClassSimt, 2, Operator_
1222 > {
1223
1224 using Shape = Shape_;
1225 using WarpShape = WarpShape_;
1226 using InstructionShape = GemmShape<1, 1, 4>;
1227 using ElementA = int8_t;
1228 using LayoutA = layout::ColumnMajor;
1229 using ElementB = int8_t;
1230 using LayoutB = layout::ColumnMajor;
1231 using ElementC = ElementC_;
1232 using LayoutC = LayoutC_;
1233 using OperatorClass = arch::OpClassSimt;
1234 static int const PartitionsK = Shape::kK / WarpShape::kK;
1235
1237 using Operator = Operator_;
1238
1240 using WarpCount = GemmShape<
1241 Shape::kM / WarpShape::kM,
1242 Shape::kN / WarpShape::kN,
1243 PartitionsK
1244 >;
1245
1246 // Divisility requirements
1247 static_assert(
1248 !(Shape::kM % WarpShape::kM) &&
1249 !(Shape::kN % WarpShape::kN),
1250 "Threadblock-scoped GEMM should be divisible by warp-scoped GEMM size."
1251 );
1252
1254 static int const kWarpSize = warp::WarpSize<arch::OpClassSimt>::value;
1255
1257 static int const kThreads = WarpCount::kCount * kWarpSize;
1258
1259 //
1260 // Shared memory layouts
1261 //
1262
1263 using SmemLayoutA = layout::ColumnMajorInterleaved<4>;
1264 using SmemLayoutB = layout::RowMajorInterleaved<4>;
1265
1266 //
1267 // Iterators to write to shared memory
1268 //
1269
1271 using IteratorThreadMapA = transform::PitchLinear2DThreadTileStripminedThreadMap<
1272 layout::PitchLinearShape<Shape::kM, Shape::kK>,
1273 kThreads,
1274 layout::PitchLinearShape<4, 4>
1275 >;
1276
1278 using SmemIteratorA = transform::threadblock::RegularTileIterator2dThreadTile<
1279 MatrixShape<Shape::kM, Shape::kK>,
1280 ElementA,
1281 SmemLayoutA,
1282 1,
1283 IteratorThreadMapA
1284 >;
1285
1286
1288 using IteratorThreadMapB = transform::PitchLinear2DThreadTileStripminedThreadMap<
1289 layout::PitchLinearShape<Shape::kK, Shape::kN>,
1290 kThreads,
1291 layout::PitchLinearShape<4, 4>
1292 >;
1293
1295 using SmemThreadMapB = transform::TransposePitchLinearThreadMap2DThreadTile<IteratorThreadMapB>;
1296
1298 using SmemIteratorB = transform::threadblock::RegularTileIterator2dThreadTile<
1299 MatrixShape<Shape::kK, Shape::kN>,
1300 ElementB,
1301 SmemLayoutB,
1302 0,
1303 SmemThreadMapB
1304 >;
1305
1306 //
1307 // Warp-level matrix multiply operator
1308 //
1309
1310 // Define the warp-level op
1311 static const int WarpNumThreadsM = detail::simt_get_warp_threads_m<WarpShape>();
1312 static const int WarpNumThreadsN = kWarpSize / WarpNumThreadsM;
1313 static const int ThreadTileM = WarpShape::kM / WarpNumThreadsM;
1314 static const int ThreadTileN = WarpShape::kN / WarpNumThreadsN;
1315 static_assert(!(WarpShape::kM % WarpNumThreadsM) && !(WarpShape::kN % WarpNumThreadsN),
1316 "WarpShape must be divisible by ThreadTile shape.");
1317 static const int LaneLayout = ThreadTileM > 4 && ThreadTileN > 4 ? 2 : 1;
1318 static const int numElementsA = 128 / sizeof_bits<ElementA>::value;
1319 static const int numElementsB = 128 / sizeof_bits<ElementB>::value;
1320 static const int LaneM = cutlass::const_min(4, ThreadTileM);
1321 static const int LaneN = cutlass::const_min(4, ThreadTileN);
1322 // these should have max of thread tile also
1323 using LaneMmaShape = cutlass::gemm::GemmShape<
1324 LaneM,
1325 LaneN,
1326 4>;
1327
1328 using Policy = cutlass::gemm::warp::MmaSimtPolicy<
1329 cutlass::MatrixShape<WarpNumThreadsM, WarpNumThreadsN>, // WarpShape
1330 cutlass::layout::ColumnMajorInterleaved<LaneLayout>, // LaneLayout
1331 LaneMmaShape
1332 >;
1333
1334 using MmaWarpSimt = cutlass::gemm::warp::MmaSimt<
1335 WarpShape,
1336 ElementA,
1337 SmemLayoutA,
1338 ElementB,
1339 SmemLayoutB,
1340 ElementC,
1341 LayoutC,
1342 Policy,
1343 PartitionsK
1344 >;
1345
1346 static int const kPaddingM = detail::simt_transpose_padding(kWarpSize, Shape::kK, sizeof_bits<ElementA>::value);
1347 static int const kPaddingN = detail::simt_transpose_padding(kWarpSize, Shape::kK, sizeof_bits<ElementB>::value);
1348
1350 using MmaPolicy = MmaPolicy<
1351 MmaWarpSimt,
1352 MatrixShape<0, 0>,
1353 MatrixShape<0, kPaddingN>,
1354 WarpCount::kK
1355 >;
1356 };
1357
1358 } // namespace threadblock
1359 } // namespace gemm
1360 } // namespace cutlass
cutlass::gemm::threadblock::DefaultMmaCore< Shape_, WarpShape_, GemmShape< 1, 1, 4 >, int8_t, layout::ColumnMajor, int8_t, layout::RowMajor, ElementC_, LayoutC_, arch::OpClassSimt, 2, Operator_ >::LayoutC
LayoutC_ LayoutC
Definition: default_mma_core_simt.h:756
mma_simt_policy.h
Describes the lane policy used by warp-level matrix multiply operators targeting SIMT instructions...
cutlass::MatrixShape
Describes the size of a matrix tile.
Definition: matrix_shape.h:42
regular_tile_iterator_pitch_linear.h
Templates implementing loading of tiles from pitch-linear rank=2 tensors.
cutlass::gemm::threadblock::DefaultMmaCore< Shape_, WarpShape_, GemmShape< 1, 1, 4 >, int8_t, layout::RowMajor, int8_t, layout::RowMajor, ElementC_, LayoutC_, arch::OpClassSimt, 2, Operator_ >::MmaPolicy
MmaPolicy< MmaWarpSimt, MatrixShape< kPaddingM, 0 >, MatrixShape< 0, 0 >, WarpCount::kK > MmaPolicy
Policy used to define MmaPipelined.
Definition: default_mma_core_simt.h:1195
cutlass
Definition: aligned_buffer.h:35
cutlass::transform::PitchLinear2DThreadTileStripminedThreadMap
Definition: pitch_linear_thread_map.h:623
cutlass::gemm::threadblock::DefaultMmaCore< Shape_, WarpShape_, GemmShape< 1, 1, 1 >, ElementA_, layout::ColumnMajor, ElementB_, layout::RowMajor, ElementC_, LayoutC_, arch::OpClassSimt, 2, Operator_ >::ElementA
ElementA_ ElementA
Definition: default_mma_core_simt.h:108
constexpr
#define constexpr
Definition: platform.h:137
cutlass::gemm::threadblock::DefaultMmaCore< Shape_, WarpShape_, GemmShape< 1, 1, 1 >, ElementA_, layout::RowMajor, ElementB_, layout::RowMajor, ElementC_, LayoutC_, arch::OpClassSimt, 2, Operator_ >::LayoutC
LayoutC_ LayoutC
Definition: default_mma_core_simt.h:435
cutlass::gemm::warp::WarpSize
Query the number of threads per warp.
Definition: gemm/warp/mma.h:43
cutlass::gemm::threadblock::DefaultMmaCore< Shape_, WarpShape_, GemmShape< 1, 1, 4 >, int8_t, layout::ColumnMajor, int8_t, layout::RowMajor, ElementC_, LayoutC_, arch::OpClassSimt, 2, Operator_ >::ElementC
ElementC_ ElementC
Definition: default_mma_core_simt.h:755
cutlass::gemm::threadblock::DefaultMmaCore
Definition: default_mma_core.h:90
cutlass::gemm::threadblock::DefaultMmaCore< Shape_, WarpShape_, GemmShape< 1, 1, 1 >, ElementA_, layout::ColumnMajor, ElementB_, layout::ColumnMajor, ElementC_, LayoutC_, arch::OpClassSimt, 2, Operator_ >::MmaPolicy
MmaPolicy< MmaWarpSimt, MatrixShape< 0, 0 >, MatrixShape< 0, kPaddingN >, WarpCount::kK > MmaPolicy
Policy used to define MmaPipelined.
Definition: default_mma_core_simt.h:719
pitch_linear_thread_map.h
Templates implementing how threads are mapped to a given tile.
cutlass::gemm::threadblock::DefaultMmaCore< Shape_, WarpShape_, GemmShape< 1, 1, 4 >, int8_t, layout::ColumnMajor, int8_t, layout::ColumnMajor, ElementC_, LayoutC_, arch::OpClassSimt, 2, Operator_ >::WarpShape
WarpShape_ WarpShape
Definition: default_mma_core_simt.h:1225
cutlass::gemm::threadblock::DefaultMmaCore< Shape_, WarpShape_, GemmShape< 1, 1, 4 >, int8_t, layout::ColumnMajor, int8_t, layout::ColumnMajor, ElementC_, LayoutC_, arch::OpClassSimt, 2, Operator_ >::LayoutC
LayoutC_ LayoutC
Definition: default_mma_core_simt.h:1232
cutlass::transform::TransposePitchLinearThreadMapSimt
Definition: pitch_linear_thread_map.h:431
cutlass::gemm::warp::MmaSimt
Structure to compute the matrix product targeting CUDA cores and SIMT math instructions.
Definition: mma_simt.h:74
cutlass::gemm::threadblock::DefaultMmaCore< Shape_, WarpShape_, GemmShape< 1, 1, 4 >, int8_t, layout::ColumnMajor, int8_t, layout::RowMajor, ElementC_, LayoutC_, arch::OpClassSimt, 2, Operator_ >::WarpShape
WarpShape_ WarpShape
Definition: default_mma_core_simt.h:749
cutlass::gemm::threadblock::DefaultMmaCore< Shape_, WarpShape_, GemmShape< 1, 1, 4 >, int8_t, layout::ColumnMajor, int8_t, layout::ColumnMajor, ElementC_, LayoutC_, arch::OpClassSimt, 2, Operator_ >::Operator
Operator_ Operator
Default Operator.
Definition: default_mma_core_simt.h:1237
cutlass::gemm::threadblock::DefaultMmaCore< Shape_, WarpShape_, GemmShape< 1, 1, 4 >, int8_t, layout::RowMajor, int8_t, layout::RowMajor, ElementC_, LayoutC_, arch::OpClassSimt, 2, Operator_ >::Operator
Operator_ Operator
Default Operator.
Definition: default_mma_core_simt.h:1078
cutlass::gemm::threadblock::DefaultMmaCore< Shape_, WarpShape_, GemmShape< 1, 1, 4 >, int8_t, layout::RowMajor, int8_t, layout::RowMajor, ElementC_, LayoutC_, arch::OpClassSimt, 2, Operator_ >::OperatorClass
arch::OpClassSimt OperatorClass
Definition: default_mma_core_simt.h:1074
cutlass::gemm::threadblock::DefaultMmaCore< Shape_, WarpShape_, GemmShape< 1, 1, 1 >, ElementA_, layout::RowMajor, ElementB_, layout::RowMajor, ElementC_, LayoutC_, arch::OpClassSimt, 2, Operator_ >::OperatorClass
arch::OpClassSimt OperatorClass
Definition: default_mma_core_simt.h:436
cutlass::gemm::threadblock::DefaultMmaCore< Shape_, WarpShape_, GemmShape< 1, 1, 1 >, ElementA_, layout::ColumnMajor, ElementB_, layout::RowMajor, ElementC_, LayoutC_, arch::OpClassSimt, 2, Operator_ >::LayoutC
LayoutC_ LayoutC
Definition: default_mma_core_simt.h:113
cutlass::gemm::threadblock::DefaultMmaCore< Shape_, WarpShape_, GemmShape< 1, 1, 4 >, int8_t, layout::ColumnMajor, int8_t, layout::RowMajor, ElementC_, LayoutC_, arch::OpClassSimt, 2, Operator_ >::Operator
Operator_ Operator
Default Operator.
Definition: default_mma_core_simt.h:761
cutlass::gemm::threadblock::DefaultMmaCore< Shape_, WarpShape_, GemmShape< 1, 1, 4 >, int8_t, layout::RowMajor, int8_t, layout::ColumnMajor, ElementC_, LayoutC_, arch::OpClassSimt, 2, Operator_ >::LayoutC
LayoutC_ LayoutC
Definition: default_mma_core_simt.h:910
cutlass::gemm::threadblock::DefaultMmaCore< Shape_, WarpShape_, GemmShape< 1, 1, 4 >, int8_t, layout::RowMajor, int8_t, layout::RowMajor, ElementC_, LayoutC_, arch::OpClassSimt, 2, Operator_ >::LayoutC
LayoutC_ LayoutC
Definition: default_mma_core_simt.h:1073
cutlass::gemm::threadblock::DefaultMmaCore< Shape_, WarpShape_, GemmShape< 1, 1, 4 >, int8_t, layout::RowMajor, int8_t, layout::ColumnMajor, ElementC_, LayoutC_, arch::OpClassSimt, 2, Operator_ >::Shape
Shape_ Shape
Definition: default_mma_core_simt.h:902
cutlass::layout::ColumnMajor
Mapping function for column-major matrices.
Definition: layout/matrix.h:142
cutlass::gemm::threadblock::DefaultMmaCore< Shape_, WarpShape_, GemmShape< 1, 1, 1 >, ElementA_, layout::ColumnMajor, ElementB_, layout::RowMajor, ElementC_, LayoutC_, arch::OpClassSimt, 2, Operator_ >::OperatorClass
arch::OpClassSimt OperatorClass
Definition: default_mma_core_simt.h:114
cutlass::layout::PitchLinearShape
Template defining a shape used by pitch-linear operators.
Definition: pitch_linear.h:43
array.h
Statically sized array of elements that accommodates all CUTLASS-supported numeric types and is safe ...
cutlass::gemm::threadblock::DefaultMmaCore< Shape_, WarpShape_, GemmShape< 1, 1, 1 >, ElementA_, layout::RowMajor, ElementB_, layout::ColumnMajor, ElementC_, LayoutC_, arch::OpClassSimt, 2, Operator_ >::OperatorClass
arch::OpClassSimt OperatorClass
Definition: default_mma_core_simt.h:270
cutlass::gemm::threadblock::DefaultMmaCore< Shape_, WarpShape_, GemmShape< 1, 1, 4 >, int8_t, layout::ColumnMajor, int8_t, layout::RowMajor, ElementC_, LayoutC_, arch::OpClassSimt, 2, Operator_ >::Shape
Shape_ Shape
Definition: default_mma_core_simt.h:748
cutlass::gemm::warp::MmaSimtPolicy
Describes the arrangement and configuration of per-lane operations in warp-level matrix multiply...
Definition: mma_simt_policy.h:46
cutlass::gemm::threadblock::DefaultMmaCore< Shape_, WarpShape_, GemmShape< 1, 1, 4 >, int8_t, layout::ColumnMajor, int8_t, layout::RowMajor, ElementC_, LayoutC_, arch::OpClassSimt, 2, Operator_ >::MmaPolicy
MmaPolicy< MmaWarpSimt, MatrixShape< 0, 0 >, MatrixShape< 0, 0 >, WarpCount::kK > MmaPolicy
Policy used to define MmaPipelined.
Definition: default_mma_core_simt.h:873
cutlass::gemm::threadblock::DefaultMmaCore< Shape_, WarpShape_, GemmShape< 1, 1, 1 >, ElementA_, layout::ColumnMajor, ElementB_, layout::RowMajor, ElementC_, LayoutC_, arch::OpClassSimt, 2, Operator_ >::WarpShape
WarpShape_ WarpShape
Definition: default_mma_core_simt.h:106
matrix_shape.h
Defines a Shape template for matrix tiles.
cutlass::gemm::threadblock::DefaultMmaCore< Shape_, WarpShape_, GemmShape< 1, 1, 4 >, int8_t, layout::ColumnMajor, int8_t, layout::ColumnMajor, ElementC_, LayoutC_, arch::OpClassSimt, 2, Operator_ >::ElementC
ElementC_ ElementC
Definition: default_mma_core_simt.h:1231
cutlass::gemm::threadblock::DefaultMmaCore< Shape_, WarpShape_, GemmShape< 1, 1, 1 >, ElementA_, layout::ColumnMajor, ElementB_, layout::RowMajor, ElementC_, LayoutC_, arch::OpClassSimt, 2, Operator_ >::Shape
Shape_ Shape
Definition: default_mma_core_simt.h:105
cutlass::gemm::threadblock::DefaultMmaCore< Shape_, WarpShape_, GemmShape< 1, 1, 4 >, int8_t, layout::RowMajor, int8_t, layout::ColumnMajor, ElementC_, LayoutC_, arch::OpClassSimt, 2, Operator_ >::MmaPolicy
MmaPolicy< MmaWarpSimt, MatrixShape< kPaddingM, 0 >, MatrixShape< 0, kPaddingN >, WarpCount::kK > MmaPolicy
Policy used to define MmaPipelined.
Definition: default_mma_core_simt.h:1036
cutlass::gemm::threadblock::DefaultMmaCore< Shape_, WarpShape_, GemmShape< 1, 1, 1 >, ElementA_, layout::RowMajor, ElementB_, layout::ColumnMajor, ElementC_, LayoutC_, arch::OpClassSimt, 2, Operator_ >::WarpShape
WarpShape_ WarpShape
Definition: default_mma_core_simt.h:262
cutlass::sizeof_bits
Defines the size of an element in bits.
Definition: numeric_types.h:42
cutlass::gemm::threadblock::DefaultMmaCore< Shape_, WarpShape_, GemmShape< 1, 1, 4 >, int8_t, layout::ColumnMajor, int8_t, layout::ColumnMajor, ElementC_, LayoutC_, arch::OpClassSimt, 2, Operator_ >::OperatorClass
arch::OpClassSimt OperatorClass
Definition: default_mma_core_simt.h:1233
cutlass::gemm::threadblock::DefaultMmaCore< Shape_, WarpShape_, GemmShape< 1, 1, 1 >, ElementA_, layout::ColumnMajor, ElementB_, layout::ColumnMajor, ElementC_, LayoutC_, arch::OpClassSimt, 2, Operator_ >::LayoutC
LayoutC_ LayoutC
Definition: default_mma_core_simt.h:597
cutlass::transform::TransposePitchLinearThreadMap2DThreadTile
Thread Mapping a 2D threadtiled mapping as a transposed Pitchlinear2DThreadTile mapping.
Definition: pitch_linear_thread_map.h:713
default_mma_core.h
Defines basic properties needed by CTA-level GEMMs assuming expectations about data layout of the glo...
cutlass::gemm::threadblock::DefaultMmaCore< Shape_, WarpShape_, GemmShape< 1, 1, 1 >, ElementA_, layout::RowMajor, ElementB_, layout::RowMajor, ElementC_, LayoutC_, arch::OpClassSimt, 2, Operator_ >::ElementC
ElementC_ ElementC
Definition: default_mma_core_simt.h:434
cutlass::gemm::threadblock::DefaultMmaCore< Shape_, WarpShape_, GemmShape< 1, 1, 4 >, int8_t, layout::RowMajor, int8_t, layout::ColumnMajor, ElementC_, LayoutC_, arch::OpClassSimt, 2, Operator_ >::WarpShape
WarpShape_ WarpShape
Definition: default_mma_core_simt.h:903
cutlass::gemm::threadblock::DefaultMmaCore< Shape_, WarpShape_, GemmShape< 1, 1, 1 >, ElementA_, layout::ColumnMajor, ElementB_, layout::RowMajor, ElementC_, LayoutC_, arch::OpClassSimt, 2, Operator_ >::Operator
Operator_ Operator
Default Operator.
Definition: default_mma_core_simt.h:118
numeric_types.h
Top-level include for all CUTLASS numeric types.
cutlass::gemm::GemmShape
Shape of a matrix multiply-add operation.
Definition: include/cutlass/gemm/gemm.h:57
cutlass::transform::threadblock::RegularTileIterator
Definition: regular_tile_iterator.h:50
static_assert
#define static_assert(__e, __m)
Definition: platform.h:153
cutlass::gemm::threadblock::DefaultMmaCore< Shape_, WarpShape_, GemmShape< 1, 1, 1 >, ElementA_, layout::ColumnMajor, ElementB_, layout::ColumnMajor, ElementC_, LayoutC_, arch::OpClassSimt, 2, Operator_ >::Operator
Operator_ Operator
Default Operator.
Definition: default_mma_core_simt.h:602
cutlass::gemm::threadblock::DefaultMmaCore< Shape_, WarpShape_, GemmShape< 1, 1, 1 >, ElementA_, layout::RowMajor, ElementB_, layout::RowMajor, ElementC_, LayoutC_, arch::OpClassSimt, 2, Operator_ >::WarpShape
WarpShape_ WarpShape
Definition: default_mma_core_simt.h:428
cutlass::gemm::threadblock::DefaultMmaCore< Shape_, WarpShape_, GemmShape< 1, 1, 4 >, int8_t, layout::RowMajor, int8_t, layout::RowMajor, ElementC_, LayoutC_, arch::OpClassSimt, 2, Operator_ >::ElementC
ElementC_ ElementC
Definition: default_mma_core_simt.h:1072
cutlass::gemm::threadblock::DefaultMmaCore< Shape_, WarpShape_, GemmShape< 1, 1, 1 >, ElementA_, layout::ColumnMajor, ElementB_, layout::ColumnMajor, ElementC_, LayoutC_, arch::OpClassSimt, 2, Operator_ >::OperatorClass
arch::OpClassSimt OperatorClass
Definition: default_mma_core_simt.h:598
cutlass::gemm::threadblock::detail::simt_transpose_padding
constexpr int simt_transpose_padding(int threads, int crosswise, int size_in_bits)
Computes padding in shared memory to perform efficient transpose without bank conflicts.
Definition: default_mma_core_simt.h:67
cutlass::gemm::threadblock::DefaultMmaCore< Shape_, WarpShape_, GemmShape< 1, 1, 4 >, int8_t, layout::RowMajor, int8_t, layout::RowMajor, ElementC_, LayoutC_, arch::OpClassSimt, 2, Operator_ >::WarpShape
WarpShape_ WarpShape
Definition: default_mma_core_simt.h:1066
cutlass::gemm::threadblock::DefaultMmaCore< Shape_, WarpShape_, GemmShape< 1, 1, 1 >, ElementA_, layout::RowMajor, ElementB_, layout::ColumnMajor, ElementC_, LayoutC_, arch::OpClassSimt, 2, Operator_ >::Shape
Shape_ Shape
Definition: default_mma_core_simt.h:261
cutlass::gemm::threadblock::DefaultMmaCore< Shape_, WarpShape_, GemmShape< 1, 1, 1 >, ElementA_, layout::ColumnMajor, ElementB_, layout::ColumnMajor, ElementC_, LayoutC_, arch::OpClassSimt, 2, Operator_ >::Shape
Shape_ Shape
Definition: default_mma_core_simt.h:589
cutlass::gemm::threadblock::DefaultMmaCore< Shape_, WarpShape_, GemmShape< 1, 1, 4 >, int8_t, layout::ColumnMajor, int8_t, layout::ColumnMajor, ElementC_, LayoutC_, arch::OpClassSimt, 2, Operator_ >::MmaPolicy
MmaPolicy< MmaWarpSimt, MatrixShape< 0, 0 >, MatrixShape< 0, kPaddingN >, WarpCount::kK > MmaPolicy
Policy used to define MmaPipelined.
Definition: default_mma_core_simt.h:1355
regular_tile_iterator_pitch_linear_2dthreadtile.h
Templates implementing loading of tiles from pitch-linear rank=2 tensors.
cutlass::layout::RowMajor
Mapping function for row-major matrices.
Definition: layout/matrix.h:50
cutlass::gemm::threadblock::detail::simt_get_warp_threads_m
constexpr int simt_get_warp_threads_m()
Definition: default_mma_core_simt.h:62
cutlass::gemm::threadblock::DefaultMmaCore< Shape_, WarpShape_, GemmShape< 1, 1, 4 >, int8_t, layout::RowMajor, int8_t, layout::ColumnMajor, ElementC_, LayoutC_, arch::OpClassSimt, 2, Operator_ >::ElementC
ElementC_ ElementC
Definition: default_mma_core_simt.h:909
cutlass::gemm::threadblock::DefaultMmaCore< Shape_, WarpShape_, GemmShape< 1, 1, 1 >, ElementA_, layout::ColumnMajor, ElementB_, layout::RowMajor, ElementC_, LayoutC_, arch::OpClassSimt, 2, Operator_ >::ElementC
ElementC_ ElementC
Definition: default_mma_core_simt.h:112
cutlass::gemm::threadblock::DefaultMmaCore< Shape_, WarpShape_, GemmShape< 1, 1, 4 >, int8_t, layout::RowMajor, int8_t, layout::RowMajor, ElementC_, LayoutC_, arch::OpClassSimt, 2, Operator_ >::Shape
Shape_ Shape
Definition: default_mma_core_simt.h:1065
cutlass::gemm::threadblock::DefaultMmaCore< Shape_, WarpShape_, GemmShape< 1, 1, 1 >, ElementA_, layout::ColumnMajor, ElementB_, layout::RowMajor, ElementC_, LayoutC_, arch::OpClassSimt, 2, Operator_ >::MmaPolicy
MmaPolicy< MmaWarpSimt, MatrixShape< 0, 0 >, MatrixShape< 0, 0 >, WarpCount::kK > MmaPolicy
Used for partial specialization.
Definition: default_mma_core_simt.h:229
cutlass::gemm::threadblock::DefaultMmaCore< Shape_, WarpShape_, GemmShape< 1, 1, 4 >, int8_t, layout::RowMajor, int8_t, layout::ColumnMajor, ElementC_, LayoutC_, arch::OpClassSimt, 2, Operator_ >::OperatorClass
arch::OpClassSimt OperatorClass
Definition: default_mma_core_simt.h:911
cutlass::gemm::threadblock::DefaultMmaCore< Shape_, WarpShape_, GemmShape< 1, 1, 1 >, ElementA_, layout::ColumnMajor, ElementB_, layout::ColumnMajor, ElementC_, LayoutC_, arch::OpClassSimt, 2, Operator_ >::ElementC
ElementC_ ElementC
Definition: default_mma_core_simt.h:596
cutlass::gemm::threadblock::DefaultMmaCore< Shape_, WarpShape_, GemmShape< 1, 1, 1 >, ElementA_, layout::RowMajor, ElementB_, layout::RowMajor, ElementC_, LayoutC_, arch::OpClassSimt, 2, Operator_ >::Operator
Operator_ Operator
Default Operator.
Definition: default_mma_core_simt.h:440
fast_math.h
Math utilities.
cutlass::layout::ColumnMajorInterleaved< 4 >
cutlass::gemm::threadblock::DefaultMmaCore< Shape_, WarpShape_, GemmShape< 1, 1, 1 >, ElementA_, layout::RowMajor, ElementB_, layout::ColumnMajor, ElementC_, LayoutC_, arch::OpClassSimt, 2, Operator_ >::MmaPolicy
MmaPolicy< MmaWarpSimt, MatrixShape< kPaddingN, 0 >, MatrixShape< 0, kPaddingN >, WarpCount::kK > MmaPolicy
Policy used to define MmaPipelined.
Definition: default_mma_core_simt.h:395
cutlass::gemm::threadblock::DefaultMmaCore< Shape_, WarpShape_, GemmShape< 1, 1, 1 >, ElementA_, layout::RowMajor, ElementB_, layout::RowMajor, ElementC_, LayoutC_, arch::OpClassSimt, 2, Operator_ >::Shape
Shape_ Shape
Definition: default_mma_core_simt.h:427
cutlass::gemm::threadblock::DefaultMmaCore< Shape_, WarpShape_, GemmShape< 1, 1, 1 >, ElementA_, layout::ColumnMajor, ElementB_, layout::RowMajor, ElementC_, LayoutC_, arch::OpClassSimt, 2, Operator_ >::ElementB
ElementB_ ElementB
Definition: default_mma_core_simt.h:110
cutlass::gemm::threadblock::DefaultMmaCore< Shape_, WarpShape_, GemmShape< 1, 1, 1 >, ElementA_, layout::RowMajor, ElementB_, layout::ColumnMajor, ElementC_, LayoutC_, arch::OpClassSimt, 2, Operator_ >::ElementC
ElementC_ ElementC
Definition: default_mma_core_simt.h:268
cutlass::gemm::threadblock::DefaultMmaCore< Shape_, WarpShape_, GemmShape< 1, 1, 1 >, ElementA_, layout::RowMajor, ElementB_, layout::ColumnMajor, ElementC_, LayoutC_, arch::OpClassSimt, 2, Operator_ >::Operator
Operator_ Operator
Default Operator.
Definition: default_mma_core_simt.h:274
mma_simt.h
Templates implementing warp-level matrix multiply-accumulate operations.
cutlass::const_min
CUTLASS_HOST_DEVICE constexpr int const_min(int a, int b)
Definition: fast_math.h:219
cutlass::gemm::threadblock::DefaultMmaCore< Shape_, WarpShape_, GemmShape< 1, 1, 4 >, int8_t, layout::RowMajor, int8_t, layout::ColumnMajor, ElementC_, LayoutC_, arch::OpClassSimt, 2, Operator_ >::Operator
Operator_ Operator
Default Operator.
Definition: default_mma_core_simt.h:915
cutlass::gemm::threadblock::DefaultMmaCore< Shape_, WarpShape_, GemmShape< 1, 1, 1 >, ElementA_, layout::RowMajor, ElementB_, layout::RowMajor, ElementC_, LayoutC_, arch::OpClassSimt, 2, Operator_ >::MmaPolicy
MmaPolicy< MmaWarpSimt, MatrixShape< kPaddingM, 0 >, MatrixShape< 0, 0 >, WarpCount::kK > MmaPolicy
Policy used to define MmaPipelined.
Definition: default_mma_core_simt.h:557
cutlass::transform::threadblock::RegularTileIterator2dThreadTile
Definition: regular_tile_iterator_pitch_linear_2dthreadtile.h:59
cutlass.h
Basic include for CUTLASS.
cutlass::transform::PitchLinearStripminedThreadMap
Definition: pitch_linear_thread_map.h:59
cutlass::gemm::threadblock::DefaultMmaCore< Shape_, WarpShape_, GemmShape< 1, 1, 1 >, ElementA_, layout::RowMajor, ElementB_, layout::ColumnMajor, ElementC_, LayoutC_, arch::OpClassSimt, 2, Operator_ >::LayoutC
LayoutC_ LayoutC
Definition: default_mma_core_simt.h:269
cutlass::gemm::threadblock::DefaultMmaCore< Shape_, WarpShape_, GemmShape< 1, 1, 4 >, int8_t, layout::ColumnMajor, int8_t, layout::RowMajor, ElementC_, LayoutC_, arch::OpClassSimt, 2, Operator_ >::OperatorClass
arch::OpClassSimt OperatorClass
Definition: default_mma_core_simt.h:757
cutlass::gemm::threadblock::DefaultMmaCore< Shape_, WarpShape_, GemmShape< 1, 1, 4 >, int8_t, layout::ColumnMajor, int8_t, layout::ColumnMajor, ElementC_, LayoutC_, arch::OpClassSimt, 2, Operator_ >::Shape
Shape_ Shape
Definition: default_mma_core_simt.h:1224
cutlass::gemm::threadblock::DefaultMmaCore< Shape_, WarpShape_, GemmShape< 1, 1, 1 >, ElementA_, layout::ColumnMajor, ElementB_, layout::ColumnMajor, ElementC_, LayoutC_, arch::OpClassSimt, 2, Operator_ >::WarpShape
WarpShape_ WarpShape
Definition: default_mma_core_simt.h:590
cutlass::layout::RowMajorInterleaved
Definition: layout/matrix.h:237