CUTLASS: tensor_op_multiplicand_sm70.h Source File

| CUTLASS CUDA Templates for Linear Algebra Subroutines and Solvers

- Main Page
- Modules
- Namespaces
- Classes
- Files

- File List
- File Members

tensor_op_multiplicand_sm70.h
Go to the documentation of this file.
1 /***************************************************************************************************
2 * Copyright (c) 2017-2019, NVIDIA CORPORATION. All rights reserved.
3 *
4 * Redistribution and use in source and binary forms, with or without modification, are permitted
5 * provided that the following conditions are met:
6 * * Redistributions of source code must retain the above copyright notice, this list of
7 * conditions and the following disclaimer.
8 * * Redistributions in binary form must reproduce the above copyright notice, this list of
9 * conditions and the following disclaimer in the documentation and/or other materials
10 * provided with the distribution.
11 * * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
12 * to endorse or promote products derived from this software without specific prior written
13 * permission.
14 *
15 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
16 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
17 * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
19 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
20 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
21 * STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
22 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
23 *
24 **************************************************************************************************/
29 #pragma once
30
31 #include "cutlass/cutlass.h"
32 #include "cutlass/coord.h"
33 #include "cutlass/layout/pitch_linear.h"
34
36
37 namespace cutlass {
38 namespace layout {
39
40 // template <
41 // int ElementSize,
42 // gemm::Operand Operand
43 // >
44 // struct VoltaTensorOpMultiplicandCongruous;
45
46 // template <
47 // int ElementSize,
48 // gemm::Operand Operand
49 // >
50 // struct ColumnMajorVoltaTensorOpMultiplicandCongruous;
51 // template <
52 // int ElementSize,
53 // gemm::Operand Operand
54 // >
55 // struct RowMajorVoltaTensorOpMultiplicandCongruous;
57
59 template <int ElementSize>
60 struct VoltaTensorOpMultiplicandCongruous {
61
63 static int const kRank = 2;
64
66 static int const kStrideRank = 1;
67
69 using Index = int32_t;
70
72 using LongIndex = int64_t;
73
75 using TensorCoord = PitchLinearCoord;
76
78 using Stride = Coord<kStrideRank, Index, LongIndex>;
79
80 //
81 // Invariants
82 //
83
85 static int const kAccessSize = 128;
86
88 using TileShape = PitchLinearShape<8, 4>;
89
91 using PartitionShape = PitchLinearShape<8, 2>;
92
93 //
94 // Static constants
95 //
96
97 static int const kElementSize = ElementSize;
98 static int const kElementsPerAccess = kAccessSize / kElementSize;
99
100 using PartitionCount = PitchLinearShape<
101 TileShape::kContiguous / PartitionShape::kContiguous,
102 TileShape::kStrided / PartitionShape::kStrided
103 >;
104
105 using AccessCount = PitchLinearShape<
106 PartitionShape::kContiguous,
107 PartitionShape::kStrided
108 >;
109
110 private:
111
112 //
113 // Data members
114 //
115
117 Stride stride_;
118
119 public:
120 //
121 // Methods
122 //
123
125 CUTLASS_HOST_DEVICE
126 VoltaTensorOpMultiplicandCongruous(Index ldm = 0): stride_(ldm) { }
127
129 CUTLASS_HOST_DEVICE
130 VoltaTensorOpMultiplicandCongruous(Stride stride): stride_(stride) { }
131
133 CUTLASS_HOST_DEVICE
134 static VoltaTensorOpMultiplicandCongruous packed(TensorCoord const &extent) {
135 return VoltaTensorOpMultiplicandCongruous(extent[0]);
136 }
137
140 CUTLASS_HOST_DEVICE
141 LongIndex operator()(TensorCoord const &coord) const {
142
143 // First, compute c and s of vector within source (in units of vector accesses)
144 int vec_contiguous_idx = coord.contiguous() / kElementsPerAccess;
145 int vec_strided_idx = coord.strided();
146
147 // Compute the fundamental tile being accessed
148 int tile_contiguous_idx = vec_contiguous_idx / TileShape::kContiguous;
149 int tile_strided_idx = vec_strided_idx / TileShape::kStrided;
150
151 int tile_contiguous_residual = vec_contiguous_idx % TileShape::kContiguous;
152 int tile_strided_residual = vec_strided_idx % TileShape::kStrided;
153
154 // Then swizzle in a tile
155 // Swizzle pattern is (tid[2:0] << 2)|(tid[4:3] ^ tid[2:1])
156 int permuted_strided_within_tile = (tile_contiguous_residual >> 1);
157 int permuted_contiguous_within_tile = (tile_strided_residual ^ permuted_strided_within_tile) |
158 ((tile_contiguous_residual & 1) << 2);
159 // Compute final element location
160 int element_contiguous = (tile_contiguous_idx * TileShape::kContiguous +
161 permuted_contiguous_within_tile) * kElementsPerAccess + (coord.contiguous() % kElementsPerAccess);
162
163 int element_strided = tile_strided_idx * TileShape::kStrided + permuted_strided_within_tile;
164
165 return element_contiguous + element_strided * stride_[0];
166 }
167
169 CUTLASS_HOST_DEVICE
170 Stride stride() const {
171 return stride_;
172 }
173
175 CUTLASS_HOST_DEVICE
176 Stride & stride() {
177 return stride_;
178 }
179
181 CUTLASS_HOST_DEVICE
182 LongIndex capacity(TensorCoord const &extent) const {
183 return extent[1] * stride_[0];
184 }
185 };
186
188
190 template <int ElementSize>
191 struct ColumnMajorVoltaTensorOpMultiplicandCongruous {
192
194 static int const kRank = 2;
195
197 static int const kStrideRank = 1;
198
200 using Index = int32_t;
201
203 using LongIndex = int64_t;
204
206 using TensorCoord = MatrixCoord;
207
209 using Stride = Coord<kStrideRank, Index, LongIndex>;
210
211 //
212 // Invariants
213 //
214
215 using Base = VoltaTensorOpMultiplicandCongruous<ElementSize>;
216
218 static int const kAccessSize = Base::kAccessSize;
219 using TileShape = typename Base::TileShape;
220 using PartitionShape = typename Base::PartitionShape;
221
222 //
223 // Static constants
224 //
225
226 static int const kElementSize = Base::kElementSize;
227 static int const kElementsPerAccess = Base::kElementsPerAccess;
228 using PartitionCount = typename Base::PartitionCount;
229 using AccessCount = typename Base::AccessCount;
230
231 private:
232
233 //
234 // Data members
235 //
236
237 Base layout_;
238
239 public:
240 //
241 // Methods
242 //
243
245 CUTLASS_HOST_DEVICE
246 ColumnMajorVoltaTensorOpMultiplicandCongruous(Index ldm = 0): layout_(ldm) { }
247
249 CUTLASS_HOST_DEVICE
250 ColumnMajorVoltaTensorOpMultiplicandCongruous(Stride stride): layout_(stride) { }
251
253 CUTLASS_HOST_DEVICE
254 static ColumnMajorVoltaTensorOpMultiplicandCongruous packed(TensorCoord const &extent) {
255 return ColumnMajorVoltaTensorOpMultiplicandCongruous(extent.row());
256 }
257
260 CUTLASS_HOST_DEVICE
261 LongIndex operator()(TensorCoord const &coord) const {
262 return layout_(PitchLinearCoord(coord.row(), coord.column()));
263 }
264
266 CUTLASS_HOST_DEVICE
267 TensorCoord inverse(LongIndex offset) const {
268 PitchLinearCoord coord = layout_.inverse(offset);
269 return MatrixCoord(coord.contiguous(), coord.strided());
270 }
271
273 CUTLASS_HOST_DEVICE
274 Stride stride() const {
275 return layout_.stride();
276 }
277
279 CUTLASS_HOST_DEVICE
280 Stride & stride() {
281 return layout_.stride();
282 }
283
285 CUTLASS_HOST_DEVICE
286 LongIndex capacity(TensorCoord const &extent) const {
287 return layout_.capacity(PitchLinearCoord(extent.row(), extent.column()));
288 }
289 };
290
292 template <int ElementSize>
293 struct RowMajorVoltaTensorOpMultiplicandCongruous {
294
296 static int const kRank = 2;
297
299 static int const kStrideRank = 1;
300
302 using Index = int32_t;
303
305 using LongIndex = int64_t;
306
308 using TensorCoord = MatrixCoord;
309
311 using Stride = Coord<kStrideRank, Index, LongIndex>;
312
313 //
314 // Invariants
315 //
316
317 using Base = VoltaTensorOpMultiplicandCongruous<ElementSize>;
318
320 static int const kAccessSize = Base::kAccessSize;
321 using TileShape = typename Base::TileShape;
322 using PartitionShape = typename Base::PartitionShape;
323
324 //
325 // Static constants
326 //
327
328 static int const kElementSize = Base::kElementSize;
329 static int const kElementsPerAccess = Base::kElementsPerAccess;
330 using PartitionCount = typename Base::PartitionCount;
331 using AccessCount = typename Base::AccessCount;
332
333 private:
334
335 //
336 // Data members
337 //
338
339 Base layout_;
340
341 public:
342 //
343 // Methods
344 //
345
347 CUTLASS_HOST_DEVICE
348 RowMajorVoltaTensorOpMultiplicandCongruous(Index ldm = 0): layout_(ldm) { }
349
351 CUTLASS_HOST_DEVICE
352 RowMajorVoltaTensorOpMultiplicandCongruous(Stride stride): layout_(stride) { }
353
355 CUTLASS_HOST_DEVICE
356 static RowMajorVoltaTensorOpMultiplicandCongruous packed(TensorCoord const &extent) {
357 return RowMajorVoltaTensorOpMultiplicandCongruous(extent.column());
358 }
359
362 CUTLASS_HOST_DEVICE
363 LongIndex operator()(TensorCoord const &coord) const {
364 return layout_(PitchLinearCoord(coord.column(), coord.row()));
365 }
366
368 CUTLASS_HOST_DEVICE
369 TensorCoord inverse(LongIndex offset) const {
370 PitchLinearCoord coord = layout_.inverse(offset);
371 return MatrixCoord(coord.strided(), coord.contiguous());
372 }
373
375 CUTLASS_HOST_DEVICE
376 Stride stride() const {
377 return layout_.stride();
378 }
379
381 CUTLASS_HOST_DEVICE
382 Stride & stride() {
383 return layout_.stride();
384 }
385
387 CUTLASS_HOST_DEVICE
388 LongIndex capacity(TensorCoord const &extent) const {
389 return layout_.capacity(PitchLinearCoord(extent.column(), extent.row()));
390 }
391 };
392
393
395 // template <int ElementSize, Operand Operand>
396 template <int ElementSize>
397 struct VoltaTensorOpMultiplicandBCongruous {
399 static int const kRank = 2;
400
402 static int const kStrideRank = 1;
403
405 using Index = int32_t;
406
408 using LongIndex = int64_t;
409
411 using TensorCoord = PitchLinearCoord;
412
414 using Stride = Coord<kStrideRank, Index, LongIndex>;
415
416 //
417 // Invariants
418 //
419
421 static int const kAccessSize = 128;
422
424 using TileShape = PitchLinearShape<8, 4>;
425
427 using PartitionShape = PitchLinearShape<4, 4>;
428
429 //
430 // Static constants
431 //
432
433 static int const kElementSize = ElementSize;
434 static int const kElementsPerAccess = kAccessSize / kElementSize;
435
436 using PartitionCount = PitchLinearShape<
437 TileShape::kContiguous / PartitionShape::kContiguous,
438 TileShape::kStrided / PartitionShape::kStrided
439 >;
440
441 using AccessCount = PitchLinearShape<
442 PartitionShape::kContiguous,
443 PartitionShape::kStrided
444 >;
445
446 private:
447
448 //
449 // Data members
450 //
451
453 Stride stride_;
454
455 public:
456 //
457 // Methods
458 //
459
461 CUTLASS_HOST_DEVICE
462 VoltaTensorOpMultiplicandBCongruous(Index ldm = 0): stride_(ldm) { }
463
465 CUTLASS_HOST_DEVICE
466 VoltaTensorOpMultiplicandBCongruous(Stride stride): stride_(stride) { }
467
469 CUTLASS_HOST_DEVICE
470 static VoltaTensorOpMultiplicandBCongruous packed(TensorCoord const &extent) {
471 return VoltaTensorOpMultiplicandBCongruous(extent[0]);
472 }
473
476 CUTLASS_HOST_DEVICE
477 LongIndex operator()(TensorCoord const &coord) const {
478
479 // First, compute c and s of vector within source (in units of vector accesses)
480 int vec_contiguous_idx = coord.contiguous() / kElementsPerAccess;
481 int vec_strided_idx = coord.strided();
482
483 // Compute the fundamental tile being accessed
484 int tile_contiguous_idx = vec_contiguous_idx / TileShape::kContiguous;
485 int tile_strided_idx = vec_strided_idx / TileShape::kStrided;
486
487 int tile_contiguous_residual = vec_contiguous_idx % TileShape::kContiguous;
488 int tile_strided_residual = vec_strided_idx % TileShape::kStrided;
489
490 // Then swizzle in a tile
491 // Swizzle pattern is (tid[1:0] << 3)|(tid & 0x4)|(tid[1:0])
492 int permuted_strided_within_tile = (tile_contiguous_residual & 0x3);
493 int permuted_contiguous_within_tile = (tile_strided_residual ^ permuted_strided_within_tile) |
494 (tile_contiguous_residual & 0x4);
495
496 // Compute final element location
497 int element_contiguous = (tile_contiguous_idx * TileShape::kContiguous +
498 permuted_contiguous_within_tile) * kElementsPerAccess + (coord.contiguous() % kElementsPerAccess);
499
500 int element_strided = tile_strided_idx * TileShape::kStrided + permuted_strided_within_tile;
501
502 return element_contiguous + element_strided * stride_[0];
503 }
504
506 CUTLASS_HOST_DEVICE
507 Stride stride() const {
508 return stride_;
509 }
510
512 CUTLASS_HOST_DEVICE
513 Stride & stride() {
514 return stride_;
515 }
516
518 CUTLASS_HOST_DEVICE
519 LongIndex capacity(TensorCoord const &extent) const {
520 return extent[1] * stride_[0];
521 }
522 };
523
525
527 template <int ElementSize>
528 struct ColumnMajorVoltaTensorOpMultiplicandBCongruous {
529
531 static int const kRank = 2;
532
534 static int const kStrideRank = 1;
535
537 using Index = int32_t;
538
540 using LongIndex = int64_t;
541
543 using TensorCoord = MatrixCoord;
544
546 using Stride = Coord<kStrideRank, Index, LongIndex>;
547
548 //
549 // Invariants
550 //
551
552 using Base = VoltaTensorOpMultiplicandBCongruous<ElementSize>;
553
555 static int const kAccessSize = Base::kAccessSize;
556 using TileShape = typename Base::TileShape;
557 using PartitionShape = typename Base::PartitionShape;
558
559 //
560 // Static constants
561 //
562
563 static int const kElementSize = Base::kElementSize;
564 static int const kElementsPerAccess = Base::kElementsPerAccess;
565 using PartitionCount = typename Base::PartitionCount;
566 using AccessCount = typename Base::AccessCount;
567
568 private:
569
570 //
571 // Data members
572 //
573
574 Base layout_;
575
576 public:
577 //
578 // Methods
579 //
580
582 CUTLASS_HOST_DEVICE
583 ColumnMajorVoltaTensorOpMultiplicandBCongruous(Index ldm = 0): layout_(ldm) { }
584
586 CUTLASS_HOST_DEVICE
587 ColumnMajorVoltaTensorOpMultiplicandBCongruous(Stride stride): layout_(stride) { }
588
590 CUTLASS_HOST_DEVICE
591 static ColumnMajorVoltaTensorOpMultiplicandBCongruous packed(TensorCoord const &extent) {
592 return ColumnMajorVoltaTensorOpMultiplicandBCongruous(extent.row());
593 }
594
597 CUTLASS_HOST_DEVICE
598 LongIndex operator()(TensorCoord const &coord) const {
599 return layout_(PitchLinearCoord(coord.row(), coord.column()));
600 }
601
603 CUTLASS_HOST_DEVICE
604 TensorCoord inverse(LongIndex offset) const {
605 PitchLinearCoord coord = layout_.inverse(offset);
606 return MatrixCoord(coord.contiguous(), coord.strided());
607 }
608
610 CUTLASS_HOST_DEVICE
611 Stride stride() const {
612 return layout_.stride();
613 }
614
616 CUTLASS_HOST_DEVICE
617 Stride & stride() {
618 return layout_.stride();
619 }
620
622 CUTLASS_HOST_DEVICE
623 LongIndex capacity(TensorCoord const &extent) const {
624 return layout_.capacity(PitchLinearCoord(extent.row(), extent.column()));
625 }
626 };
627
629 template <int ElementSize>
630 struct RowMajorVoltaTensorOpMultiplicandBCongruous {
631
633 static int const kRank = 2;
634
636 static int const kStrideRank = 1;
637
639 using Index = int32_t;
640
642 using LongIndex = int64_t;
643
645 using TensorCoord = MatrixCoord;
646
648 using Stride = Coord<kStrideRank, Index, LongIndex>;
649
650 //
651 // Invariants
652 //
653
654 using Base = VoltaTensorOpMultiplicandBCongruous<ElementSize>;
655
657 static int const kAccessSize = Base::kAccessSize;
658 using TileShape = typename Base::TileShape;
659 using PartitionShape = typename Base::PartitionShape;
660
661 //
662 // Static constants
663 //
664
665 static int const kElementSize = Base::kElementSize;
666 static int const kElementsPerAccess = Base::kElementsPerAccess;
667 using PartitionCount = typename Base::PartitionCount;
668 using AccessCount = typename Base::AccessCount;
669
670 private:
671
672 //
673 // Data members
674 //
675
676 Base layout_;
677
678 public:
679 //
680 // Methods
681 //
682
684 CUTLASS_HOST_DEVICE
685 RowMajorVoltaTensorOpMultiplicandBCongruous(Index ldm = 0): layout_(ldm) { }
686
688 CUTLASS_HOST_DEVICE
689 RowMajorVoltaTensorOpMultiplicandBCongruous(Stride stride): layout_(stride) { }
690
692 CUTLASS_HOST_DEVICE
693 static RowMajorVoltaTensorOpMultiplicandBCongruous packed(TensorCoord const &extent) {
694 return RowMajorVoltaTensorOpMultiplicandBCongruous(extent.column());
695 }
696
699 CUTLASS_HOST_DEVICE
700 LongIndex operator()(TensorCoord const &coord) const {
701 return layout_(PitchLinearCoord(coord.column(), coord.row()));
702 }
703
705 CUTLASS_HOST_DEVICE
706 TensorCoord inverse(LongIndex offset) const {
707 PitchLinearCoord coord = layout_.inverse(offset);
708 return MatrixCoord(coord.strided(), coord.contiguous());
709 }
710
712 CUTLASS_HOST_DEVICE
713 Stride stride() const {
714 return layout_.stride();
715 }
716
718 CUTLASS_HOST_DEVICE
719 Stride & stride() {
720 return layout_.stride();
721 }
722
724 CUTLASS_HOST_DEVICE
725 LongIndex capacity(TensorCoord const &extent) const {
726 return layout_.capacity(PitchLinearCoord(extent.column(), extent.row()));
727 }
728 };
729
732 template <int ElementSize, int KBlock>
733 struct VoltaTensorOpMultiplicandCrosswise {
735 static int const kRank = 2;
736
738 static int const kStrideRank = 1;
739
741 using Index = int32_t;
742
744 using LongIndex = int64_t;
745
747 using TensorCoord = PitchLinearCoord;
748
750 using Stride = Coord<kStrideRank, Index, LongIndex>;
751
752 //
753 // Invariants
754 //
755
757 static int const kAccessSize = 64;
758
759 //
760 // Static constants
761 //
762
763 static int const kElementSize = ElementSize;
764 static int const kElementsPerAccess = kAccessSize / kElementSize;
765 static int const kKBlock = KBlock;
766
767 private:
768 //
769 // Data members
770 //
771
773 Stride stride_;
774 public:
775 //
776 // Methods
777 //
778
780 CUTLASS_HOST_DEVICE
781 VoltaTensorOpMultiplicandCrosswise(Index ldm = 0) : stride_(ldm) {}
782
784 CUTLASS_HOST_DEVICE
785 VoltaTensorOpMultiplicandCrosswise(Stride stride) : stride_(stride) {}
786
788 CUTLASS_HOST_DEVICE
789 static VoltaTensorOpMultiplicandCrosswise packed(TensorCoord const &extent) {
790 return VoltaTensorOpMultiplicandCrosswise(extent[1]);
791 }
792
795 CUTLASS_HOST_DEVICE
796 LongIndex operator()(TensorCoord const &coord) const {
797
798 //
799 // First, compute c and s of vector within source (in units of vector
800 // accesses)
801 //
802 int vec_contiguous_idx = coord.contiguous() / kElementsPerAccess;
803 int vec_strided_idx = coord.strided();
804
805 //
806 // Then swizzle
807 // The mapping is like this:
808 // id[1:0]|(id[3]^id[4])|id[2]
809
810 int vec_strided_within_tile = vec_contiguous_idx & 0x7;
811 int permuted_vec_contiguous =
812 (vec_strided_idx & (~0xF)) + (vec_strided_idx & 0x3) * 4 +
813 (((vec_strided_idx >> 2) ^ ((vec_strided_idx & 0x10) >> 3)) & 0x3);
814
815 permuted_vec_contiguous ^= ((vec_strided_within_tile >> 1) & 0x3);
816
817 int permuted_vec_strided = vec_contiguous_idx;
818
819 //
820 // Compute final element location
821 //
822
823 int element_contiguous = permuted_vec_contiguous * kElementsPerAccess +
824 (coord.contiguous() % kElementsPerAccess);
825
826 return element_contiguous + permuted_vec_strided * (stride_[0] * kElementsPerAccess);
827 }
828
830 CUTLASS_HOST_DEVICE
831 Stride stride() const { return stride_; }
832
834 CUTLASS_HOST_DEVICE
835 Stride &stride() { return stride_; }
836
839 CUTLASS_HOST_DEVICE
840 LongIndex capacity(TensorCoord const &extent) const {
841 return extent[0] * stride_[0];
842 }
843 };
844
847 template <int ElementSize, int KBlock>
848 struct ColumnMajorVoltaTensorOpMultiplicandCrosswise {
850 static int const kRank = 2;
851
853 static int const kStrideRank = 1;
854
856 using Index = int32_t;
857
859 using LongIndex = int64_t;
860
862 using TensorCoord = MatrixCoord;
863
865 using Stride = Coord<kStrideRank, Index, LongIndex>;
866
867 //
868 // Invariants
869 //
870
871 using Base = VoltaTensorOpMultiplicandCrosswise<ElementSize, KBlock>;
872
874 static int const kAccessSize = Base::kAccessSize;
875
876 //
877 // Static constants
878 //
879
880 static int const kElementSize = Base::kElementSize;
881 static int const kElementsPerAccess = Base::kElementsPerAccess;
882
883 private:
884 //
885 // Data members
886 //
887
888 Base layout_;
889
890 public:
891 //
892 // Methods
893 //
894
896 CUTLASS_HOST_DEVICE
897 ColumnMajorVoltaTensorOpMultiplicandCrosswise(Index ldm = 0) : layout_(ldm) {}
898
900 CUTLASS_HOST_DEVICE
901 ColumnMajorVoltaTensorOpMultiplicandCrosswise(Stride stride) : layout_(stride) {}
902
904 CUTLASS_HOST_DEVICE
905 static ColumnMajorVoltaTensorOpMultiplicandCrosswise packed(
906 TensorCoord const &extent) {
907 return ColumnMajorVoltaTensorOpMultiplicandCrosswise(extent.column());
908 }
909
912 CUTLASS_HOST_DEVICE
913 LongIndex operator()(TensorCoord const &coord) const {
914 return layout_(PitchLinearCoord(coord.row(), coord.column()));
915 }
916
918 CUTLASS_HOST_DEVICE
919 TensorCoord inverse(LongIndex offset) const {
920 PitchLinearCoord coord = layout_.inverse(offset);
921 return MatrixCoord(coord.contiguous(), coord.strided());
922 }
923
925 CUTLASS_HOST_DEVICE
926 Stride stride() const { return layout_.stride(); }
927
929 CUTLASS_HOST_DEVICE
930 Stride &stride() { return layout_.stride(); }
931
934 CUTLASS_HOST_DEVICE
935 LongIndex capacity(TensorCoord const &extent) const {
936 return layout_.capacity(PitchLinearCoord(extent.row(), extent.column()));
937 }
938 };
939
942 template <int ElementSize, int KBlock>
943 struct RowMajorVoltaTensorOpMultiplicandCrosswise {
945 static int const kRank = 2;
946
948 static int const kStrideRank = 1;
949
951 using Index = int32_t;
952
954 using LongIndex = int64_t;
955
957 using TensorCoord = MatrixCoord;
958
960 using Stride = Coord<kStrideRank, Index, LongIndex>;
961
962 //
963 // Invariants
964 //
965
966 using Base = VoltaTensorOpMultiplicandCrosswise<ElementSize, KBlock>;
967
969 static int const kAccessSize = Base::kAccessSize;
970
971 //
972 // Static constants
973 //
974
975 static int const kElementSize = Base::kElementSize;
976 static int const kElementsPerAccess = Base::kElementsPerAccess;
977
978 private:
979 //
980 // Data members
981 //
982
983 Base layout_;
984
985 public:
986 //
987 // Methods
988 //
989
991 CUTLASS_HOST_DEVICE
992 RowMajorVoltaTensorOpMultiplicandCrosswise(Index ldm = 0) : layout_(ldm) {}
993
995 CUTLASS_HOST_DEVICE
996 RowMajorVoltaTensorOpMultiplicandCrosswise(Stride stride) : layout_(stride) {}
997
999 CUTLASS_HOST_DEVICE
1000 static RowMajorVoltaTensorOpMultiplicandCrosswise packed(
1001 TensorCoord const &extent) {
1002 return RowMajorVoltaTensorOpMultiplicandCrosswise(extent.row());
1003 }
1004
1007 CUTLASS_HOST_DEVICE
1008 LongIndex operator()(TensorCoord const &coord) const {
1009 return layout_(PitchLinearCoord(coord.column(), coord.row()));
1010 }
1011
1013 CUTLASS_HOST_DEVICE
1014 TensorCoord inverse(LongIndex offset) const {
1015 PitchLinearCoord coord = layout_.inverse(offset);
1016 return MatrixCoord(coord.strided(), coord.contiguous());
1017 }
1018
1020 CUTLASS_HOST_DEVICE
1021 Stride stride() const { return layout_.stride(); }
1022
1024 CUTLASS_HOST_DEVICE
1025 Stride &stride() { return layout_.stride(); }
1026
1029 CUTLASS_HOST_DEVICE
1030 LongIndex capacity(TensorCoord const &extent) const {
1031 return layout_.capacity(PitchLinearCoord(extent.column(), extent.row()));
1032 }
1033 };
1034
1035 } // namespace layout
1036 } // namespace cutlass
1037
cutlass::layout::RowMajorVoltaTensorOpMultiplicandCongruous::capacity
CUTLASS_HOST_DEVICE LongIndex capacity(TensorCoord const &extent) const
Compute the number of contiguous elements needed to store a tensor with the given size...
Definition: tensor_op_multiplicand_sm70.h:388
cutlass::layout::ColumnMajorVoltaTensorOpMultiplicandCrosswise::capacity
CUTLASS_HOST_DEVICE LongIndex capacity(TensorCoord const &extent) const
Definition: tensor_op_multiplicand_sm70.h:935
cutlass::layout::RowMajorVoltaTensorOpMultiplicandCrosswise::stride
CUTLASS_HOST_DEVICE Stride stride() const
Returns the stride of the layout.
Definition: tensor_op_multiplicand_sm70.h:1021
cutlass::layout::RowMajorVoltaTensorOpMultiplicandBCongruous
Template mapping a row-major view of pitch-linear memory to VoltaTensorOpMultiplicandCongruous.
Definition: tensor_op_multiplicand_sm70.h:630
cutlass::layout::ColumnMajorVoltaTensorOpMultiplicandBCongruous::Index
int32_t Index
Index type used for coordinates.
Definition: tensor_op_multiplicand_sm70.h:537
cutlass::layout::VoltaTensorOpMultiplicandBCongruous::capacity
CUTLASS_HOST_DEVICE LongIndex capacity(TensorCoord const &extent) const
Compute the number of contiguous elements needed to store a tensor with the given size...
Definition: tensor_op_multiplicand_sm70.h:519
cutlass::layout::ColumnMajorVoltaTensorOpMultiplicandCongruous::operator()
CUTLASS_HOST_DEVICE LongIndex operator()(TensorCoord const &coord) const
Definition: tensor_op_multiplicand_sm70.h:261
cutlass::MatrixCoord::column
CUTLASS_HOST_DEVICE Index const & column() const
Returns the column of the coordinate.
Definition: matrix_coord.h:85
cutlass::layout::ColumnMajorVoltaTensorOpMultiplicandBCongruous::inverse
CUTLASS_HOST_DEVICE TensorCoord inverse(LongIndex offset) const
Inverse of layout function, mapping linear offset to logical coordinate.
Definition: tensor_op_multiplicand_sm70.h:604
cutlass::layout::RowMajorVoltaTensorOpMultiplicandCrosswise::RowMajorVoltaTensorOpMultiplicandCrosswise
CUTLASS_HOST_DEVICE RowMajorVoltaTensorOpMultiplicandCrosswise(Index ldm=0)
Ctor.
Definition: tensor_op_multiplicand_sm70.h:992
cutlass::layout::VoltaTensorOpMultiplicandCongruous::Index
int32_t Index
Index type used for coordinates.
Definition: tensor_op_multiplicand_sm70.h:69
cutlass
Definition: aligned_buffer.h:35
cutlass::layout::VoltaTensorOpMultiplicandCongruous::VoltaTensorOpMultiplicandCongruous
CUTLASS_HOST_DEVICE VoltaTensorOpMultiplicandCongruous(Stride stride)
Ctor.
Definition: tensor_op_multiplicand_sm70.h:130
cutlass::layout::PitchLinearCoord
Coordinate in pitch-linear space.
Definition: pitch_linear.h:52
cutlass::layout::ColumnMajorVoltaTensorOpMultiplicandBCongruous::stride
CUTLASS_HOST_DEVICE Stride & stride()
Returns the stride of the layout.
Definition: tensor_op_multiplicand_sm70.h:617
cutlass::layout::ColumnMajorVoltaTensorOpMultiplicandCongruous::PartitionCount
typename Base::PartitionCount PartitionCount
Definition: tensor_op_multiplicand_sm70.h:228
cutlass::layout::ColumnMajorVoltaTensorOpMultiplicandCongruous::PartitionShape
typename Base::PartitionShape PartitionShape
Definition: tensor_op_multiplicand_sm70.h:220
cutlass::layout::RowMajorVoltaTensorOpMultiplicandCongruous::packed
static CUTLASS_HOST_DEVICE RowMajorVoltaTensorOpMultiplicandCongruous packed(TensorCoord const &extent)
Helper returns a layout to a tightly packed tensor.
Definition: tensor_op_multiplicand_sm70.h:356
cutlass::layout::ColumnMajorVoltaTensorOpMultiplicandCrosswise::stride
CUTLASS_HOST_DEVICE Stride & stride()
Returns the stride of the layout.
Definition: tensor_op_multiplicand_sm70.h:930
coord.h
A Coord is a coordinate of arbitrary rank into a tensor or matrix.
cutlass::layout::ColumnMajorVoltaTensorOpMultiplicandCrosswise::LongIndex
int64_t LongIndex
Long index type used for offsets.
Definition: tensor_op_multiplicand_sm70.h:859
cutlass::layout::RowMajorVoltaTensorOpMultiplicandBCongruous::inverse
CUTLASS_HOST_DEVICE TensorCoord inverse(LongIndex offset) const
Inverse of layout function, mapping linear offset to logical coordinate.
Definition: tensor_op_multiplicand_sm70.h:706
cutlass::layout::RowMajorVoltaTensorOpMultiplicandCrosswise::capacity
CUTLASS_HOST_DEVICE LongIndex capacity(TensorCoord const &extent) const
Definition: tensor_op_multiplicand_sm70.h:1030
cutlass::layout::VoltaTensorOpMultiplicandCongruous::stride
CUTLASS_HOST_DEVICE Stride stride() const
Returns the stride of the layout.
Definition: tensor_op_multiplicand_sm70.h:170
cutlass::layout::RowMajorVoltaTensorOpMultiplicandCongruous::stride
CUTLASS_HOST_DEVICE Stride & stride()
Returns the stride of the layout.
Definition: tensor_op_multiplicand_sm70.h:382
cutlass::layout::ColumnMajorVoltaTensorOpMultiplicandBCongruous::ColumnMajorVoltaTensorOpMultiplicandBCongruous
CUTLASS_HOST_DEVICE ColumnMajorVoltaTensorOpMultiplicandBCongruous(Index ldm=0)
Ctor.
Definition: tensor_op_multiplicand_sm70.h:583
cutlass::layout::RowMajorVoltaTensorOpMultiplicandCongruous::AccessCount
typename Base::AccessCount AccessCount
Definition: tensor_op_multiplicand_sm70.h:331
cutlass::layout::ColumnMajorVoltaTensorOpMultiplicandCrosswise::stride
CUTLASS_HOST_DEVICE Stride stride() const
Returns the stride of the layout.
Definition: tensor_op_multiplicand_sm70.h:926
cutlass::MatrixCoord::row
CUTLASS_HOST_DEVICE Index const & row() const
Returns the row of the coordinate.
Definition: matrix_coord.h:77
cutlass::layout::ColumnMajorVoltaTensorOpMultiplicandBCongruous::LongIndex
int64_t LongIndex
Long index type used for offsets.
Definition: tensor_op_multiplicand_sm70.h:540
cutlass::layout::ColumnMajorVoltaTensorOpMultiplicandCrosswise
Definition: tensor_op_multiplicand_sm70.h:848
cutlass::layout::VoltaTensorOpMultiplicandBCongruous::operator()
CUTLASS_HOST_DEVICE LongIndex operator()(TensorCoord const &coord) const
Definition: tensor_op_multiplicand_sm70.h:477
cutlass::layout::RowMajorVoltaTensorOpMultiplicandCrosswise::RowMajorVoltaTensorOpMultiplicandCrosswise
CUTLASS_HOST_DEVICE RowMajorVoltaTensorOpMultiplicandCrosswise(Stride stride)
Ctor.
Definition: tensor_op_multiplicand_sm70.h:996
cutlass::layout::VoltaTensorOpMultiplicandBCongruous::stride
CUTLASS_HOST_DEVICE Stride stride() const
Returns the stride of the layout.
Definition: tensor_op_multiplicand_sm70.h:507
cutlass::layout::VoltaTensorOpMultiplicandCongruous::kElementSize
static int const kElementSize
Definition: tensor_op_multiplicand_sm70.h:97
cutlass::layout::RowMajorVoltaTensorOpMultiplicandBCongruous::TileShape
typename Base::TileShape TileShape
Definition: tensor_op_multiplicand_sm70.h:658
cutlass::layout::VoltaTensorOpMultiplicandCrosswise::capacity
CUTLASS_HOST_DEVICE LongIndex capacity(TensorCoord const &extent) const
Definition: tensor_op_multiplicand_sm70.h:840
cutlass::layout::RowMajorVoltaTensorOpMultiplicandCongruous::TileShape
typename Base::TileShape TileShape
Definition: tensor_op_multiplicand_sm70.h:321
cutlass::layout::RowMajorVoltaTensorOpMultiplicandBCongruous::packed
static CUTLASS_HOST_DEVICE RowMajorVoltaTensorOpMultiplicandBCongruous packed(TensorCoord const &extent)
Helper returns a layout to a tightly packed tensor.
Definition: tensor_op_multiplicand_sm70.h:693
cutlass::layout::ColumnMajorVoltaTensorOpMultiplicandCongruous::inverse
CUTLASS_HOST_DEVICE TensorCoord inverse(LongIndex offset) const
Inverse of layout function, mapping linear offset to logical coordinate.
Definition: tensor_op_multiplicand_sm70.h:267
cutlass::layout::RowMajorVoltaTensorOpMultiplicandBCongruous::LongIndex
int64_t LongIndex
Long index type used for offsets.
Definition: tensor_op_multiplicand_sm70.h:642
cutlass::layout::VoltaTensorOpMultiplicandCongruous::packed
static CUTLASS_HOST_DEVICE VoltaTensorOpMultiplicandCongruous packed(TensorCoord const &extent)
Helper returns a layout to a tightly packed tensor.
Definition: tensor_op_multiplicand_sm70.h:134
cutlass::layout::ColumnMajorVoltaTensorOpMultiplicandBCongruous::PartitionShape
typename Base::PartitionShape PartitionShape
Definition: tensor_op_multiplicand_sm70.h:557
cutlass::layout::PitchLinearShape
Template defining a shape used by pitch-linear operators.
Definition: pitch_linear.h:43
cutlass::layout::RowMajorVoltaTensorOpMultiplicandCrosswise::stride
CUTLASS_HOST_DEVICE Stride & stride()
Returns the stride of the layout.
Definition: tensor_op_multiplicand_sm70.h:1025
cutlass::layout::RowMajorVoltaTensorOpMultiplicandCongruous::inverse
CUTLASS_HOST_DEVICE TensorCoord inverse(LongIndex offset) const
Inverse of layout function, mapping linear offset to logical coordinate.
Definition: tensor_op_multiplicand_sm70.h:369
cutlass::layout::ColumnMajorVoltaTensorOpMultiplicandCongruous::Index
int32_t Index
Index type used for coordinates.
Definition: tensor_op_multiplicand_sm70.h:200
cutlass::layout::RowMajorVoltaTensorOpMultiplicandCongruous::RowMajorVoltaTensorOpMultiplicandCongruous
CUTLASS_HOST_DEVICE RowMajorVoltaTensorOpMultiplicandCongruous(Stride stride)
Ctor.
Definition: tensor_op_multiplicand_sm70.h:352
cutlass::layout::ColumnMajorVoltaTensorOpMultiplicandCongruous::TileShape
typename Base::TileShape TileShape
Definition: tensor_op_multiplicand_sm70.h:219
cutlass::layout::VoltaTensorOpMultiplicandBCongruous::stride
CUTLASS_HOST_DEVICE Stride & stride()
Returns the stride of the layout.
Definition: tensor_op_multiplicand_sm70.h:513
cutlass::layout::ColumnMajorVoltaTensorOpMultiplicandBCongruous
Template mapping a column-major view of pitch-linear memory to VoltaTensorOpMultiplicandCongruous.
Definition: tensor_op_multiplicand_sm70.h:528
cutlass::layout::PitchLinearShape::kStrided
static int const kStrided
Definition: pitch_linear.h:45
cutlass::layout::VoltaTensorOpMultiplicandCrosswise::VoltaTensorOpMultiplicandCrosswise
CUTLASS_HOST_DEVICE VoltaTensorOpMultiplicandCrosswise(Stride stride)
Ctor.
Definition: tensor_op_multiplicand_sm70.h:785
cutlass::layout::ColumnMajorVoltaTensorOpMultiplicandBCongruous::capacity
CUTLASS_HOST_DEVICE LongIndex capacity(TensorCoord const &extent) const
Compute the number of contiguous elements needed to store a tensor with the given size...
Definition: tensor_op_multiplicand_sm70.h:623
cutlass::layout::RowMajorVoltaTensorOpMultiplicandCrosswise::operator()
CUTLASS_HOST_DEVICE LongIndex operator()(TensorCoord const &coord) const
Definition: tensor_op_multiplicand_sm70.h:1008
cutlass::layout::VoltaTensorOpMultiplicandBCongruous::Index
int32_t Index
Index type used for coordinates.
Definition: tensor_op_multiplicand_sm70.h:405
cutlass::layout::PitchLinearShape::kContiguous
static int const kContiguous
Definition: pitch_linear.h:44
cutlass::layout::RowMajorVoltaTensorOpMultiplicandBCongruous::PartitionCount
typename Base::PartitionCount PartitionCount
Definition: tensor_op_multiplicand_sm70.h:667
cutlass::layout::VoltaTensorOpMultiplicandCongruous::kElementsPerAccess
static int const kElementsPerAccess
Definition: tensor_op_multiplicand_sm70.h:98
cutlass::layout::ColumnMajorVoltaTensorOpMultiplicandCrosswise::inverse
CUTLASS_HOST_DEVICE TensorCoord inverse(LongIndex offset) const
Inverse of layout function, mapping linear offset to logical coordinate.
Definition: tensor_op_multiplicand_sm70.h:919
cutlass::layout::ColumnMajorVoltaTensorOpMultiplicandBCongruous::packed
static CUTLASS_HOST_DEVICE ColumnMajorVoltaTensorOpMultiplicandBCongruous packed(TensorCoord const &extent)
Helper returns a layout to a tightly packed tensor.
Definition: tensor_op_multiplicand_sm70.h:591
cutlass::layout::ColumnMajorVoltaTensorOpMultiplicandCongruous::LongIndex
int64_t LongIndex
Long index type used for offsets.
Definition: tensor_op_multiplicand_sm70.h:203
cutlass::layout::RowMajorVoltaTensorOpMultiplicandCongruous::PartitionCount
typename Base::PartitionCount PartitionCount
Definition: tensor_op_multiplicand_sm70.h:330
cutlass::layout::RowMajorVoltaTensorOpMultiplicandBCongruous::capacity
CUTLASS_HOST_DEVICE LongIndex capacity(TensorCoord const &extent) const
Compute the number of contiguous elements needed to store a tensor with the given size...
Definition: tensor_op_multiplicand_sm70.h:725
cutlass::layout::ColumnMajorVoltaTensorOpMultiplicandBCongruous::operator()
CUTLASS_HOST_DEVICE LongIndex operator()(TensorCoord const &coord) const
Definition: tensor_op_multiplicand_sm70.h:598
cutlass::layout::ColumnMajorVoltaTensorOpMultiplicandBCongruous::PartitionCount
typename Base::PartitionCount PartitionCount
Definition: tensor_op_multiplicand_sm70.h:565
cutlass::layout::RowMajorVoltaTensorOpMultiplicandCrosswise::Index
int32_t Index
Index type used for coordinates.
Definition: tensor_op_multiplicand_sm70.h:951
cutlass::layout::RowMajorVoltaTensorOpMultiplicandCongruous
Template mapping a row-major view of pitch-linear memory to VoltaTensorOpMultiplicandCongruous.
Definition: tensor_op_multiplicand_sm70.h:293
cutlass::layout::VoltaTensorOpMultiplicandCongruous::kRank
static int const kRank
Logical rank of tensor.
Definition: tensor_op_multiplicand_sm70.h:63
cutlass::layout::RowMajorVoltaTensorOpMultiplicandCrosswise::packed
static CUTLASS_HOST_DEVICE RowMajorVoltaTensorOpMultiplicandCrosswise packed(TensorCoord const &extent)
Helper returns a layout to a tightly packed tensor.
Definition: tensor_op_multiplicand_sm70.h:1000
cutlass::layout::RowMajorVoltaTensorOpMultiplicandCongruous::LongIndex
int64_t LongIndex
Long index type used for offsets.
Definition: tensor_op_multiplicand_sm70.h:305
cutlass::layout::RowMajorVoltaTensorOpMultiplicandBCongruous::PartitionShape
typename Base::PartitionShape PartitionShape
Definition: tensor_op_multiplicand_sm70.h:659
cutlass::layout::RowMajorVoltaTensorOpMultiplicandCrosswise::inverse
CUTLASS_HOST_DEVICE TensorCoord inverse(LongIndex offset) const
Inverse of layout function, mapping linear offset to logical coordinate.
Definition: tensor_op_multiplicand_sm70.h:1014
cutlass::layout::ColumnMajorVoltaTensorOpMultiplicandCrosswise::ColumnMajorVoltaTensorOpMultiplicandCrosswise
CUTLASS_HOST_DEVICE ColumnMajorVoltaTensorOpMultiplicandCrosswise(Index ldm=0)
Ctor.
Definition: tensor_op_multiplicand_sm70.h:897
cutlass::layout::VoltaTensorOpMultiplicandBCongruous
Template based on element size (in bits) - defined in terms of pitch-linear memory.
Definition: tensor_op_multiplicand_sm70.h:397
cutlass::layout::RowMajorVoltaTensorOpMultiplicandBCongruous::AccessCount
typename Base::AccessCount AccessCount
Definition: tensor_op_multiplicand_sm70.h:668
cutlass::layout::RowMajorVoltaTensorOpMultiplicandCongruous::operator()
CUTLASS_HOST_DEVICE LongIndex operator()(TensorCoord const &coord) const
Definition: tensor_op_multiplicand_sm70.h:363
cutlass::layout::ColumnMajorVoltaTensorOpMultiplicandCrosswise::operator()
CUTLASS_HOST_DEVICE LongIndex operator()(TensorCoord const &coord) const
Definition: tensor_op_multiplicand_sm70.h:913
CUTLASS_HOST_DEVICE
#define CUTLASS_HOST_DEVICE
Definition: cutlass.h:89
cutlass::layout::RowMajorVoltaTensorOpMultiplicandCongruous::Index
int32_t Index
Index type used for coordinates.
Definition: tensor_op_multiplicand_sm70.h:302
cutlass::layout::VoltaTensorOpMultiplicandBCongruous::VoltaTensorOpMultiplicandBCongruous
CUTLASS_HOST_DEVICE VoltaTensorOpMultiplicandBCongruous(Stride stride)
Ctor.
Definition: tensor_op_multiplicand_sm70.h:466
cutlass::layout::VoltaTensorOpMultiplicandCrosswise::packed
static CUTLASS_HOST_DEVICE VoltaTensorOpMultiplicandCrosswise packed(TensorCoord const &extent)
Helper returns a layout to a tightly packed tensor.
Definition: tensor_op_multiplicand_sm70.h:789
cutlass::layout::PitchLinearCoord::contiguous
CUTLASS_HOST_DEVICE Index const & contiguous() const
Returns the contiguous dimension.
Definition: pitch_linear.h:89
cutlass::layout::RowMajorVoltaTensorOpMultiplicandBCongruous::RowMajorVoltaTensorOpMultiplicandBCongruous
CUTLASS_HOST_DEVICE RowMajorVoltaTensorOpMultiplicandBCongruous(Stride stride)
Ctor.
Definition: tensor_op_multiplicand_sm70.h:689
cutlass::layout::VoltaTensorOpMultiplicandBCongruous::packed
static CUTLASS_HOST_DEVICE VoltaTensorOpMultiplicandBCongruous packed(TensorCoord const &extent)
Helper returns a layout to a tightly packed tensor.
Definition: tensor_op_multiplicand_sm70.h:470
cutlass::layout::VoltaTensorOpMultiplicandCongruous::LongIndex
int64_t LongIndex
Long index type used for offsets.
Definition: tensor_op_multiplicand_sm70.h:72
cutlass::layout::ColumnMajorVoltaTensorOpMultiplicandCongruous::packed
static CUTLASS_HOST_DEVICE ColumnMajorVoltaTensorOpMultiplicandCongruous packed(TensorCoord const &extent)
Helper returns a layout to a tightly packed tensor.
Definition: tensor_op_multiplicand_sm70.h:254
cutlass::layout::VoltaTensorOpMultiplicandCrosswise::operator()
CUTLASS_HOST_DEVICE LongIndex operator()(TensorCoord const &coord) const
Definition: tensor_op_multiplicand_sm70.h:796
cutlass::layout::VoltaTensorOpMultiplicandCrosswise::Index
int32_t Index
Index type used for coordinates.
Definition: tensor_op_multiplicand_sm70.h:741
cutlass::layout::RowMajorVoltaTensorOpMultiplicandBCongruous::stride
CUTLASS_HOST_DEVICE Stride & stride()
Returns the stride of the layout.
Definition: tensor_op_multiplicand_sm70.h:719
cutlass::layout::ColumnMajorVoltaTensorOpMultiplicandBCongruous::stride
CUTLASS_HOST_DEVICE Stride stride() const
Returns the stride of the layout.
Definition: tensor_op_multiplicand_sm70.h:611
cutlass::layout::ColumnMajorVoltaTensorOpMultiplicandCrosswise::packed
static CUTLASS_HOST_DEVICE ColumnMajorVoltaTensorOpMultiplicandCrosswise packed(TensorCoord const &extent)
Helper returns a layout to a tightly packed tensor.
Definition: tensor_op_multiplicand_sm70.h:905
cutlass::layout::VoltaTensorOpMultiplicandCrosswise::stride
CUTLASS_HOST_DEVICE Stride & stride()
Returns the stride of the layout.
Definition: tensor_op_multiplicand_sm70.h:835
cutlass::layout::ColumnMajorVoltaTensorOpMultiplicandCongruous::stride
CUTLASS_HOST_DEVICE Stride stride() const
Returns the stride of the layout.
Definition: tensor_op_multiplicand_sm70.h:274
cutlass::layout::ColumnMajorVoltaTensorOpMultiplicandBCongruous::AccessCount
typename Base::AccessCount AccessCount
Definition: tensor_op_multiplicand_sm70.h:566
cutlass::layout::VoltaTensorOpMultiplicandCrosswise
Definition: tensor_op_multiplicand_sm70.h:733
cutlass::layout::RowMajorVoltaTensorOpMultiplicandCongruous::RowMajorVoltaTensorOpMultiplicandCongruous
CUTLASS_HOST_DEVICE RowMajorVoltaTensorOpMultiplicandCongruous(Index ldm=0)
Ctor.
Definition: tensor_op_multiplicand_sm70.h:348
cutlass::Coord< kStrideRank, Index, LongIndex >
cutlass::layout::RowMajorVoltaTensorOpMultiplicandCrosswise
Definition: tensor_op_multiplicand_sm70.h:943
cutlass::layout::ColumnMajorVoltaTensorOpMultiplicandBCongruous::TileShape
typename Base::TileShape TileShape
Definition: tensor_op_multiplicand_sm70.h:556
cutlass::layout::RowMajorVoltaTensorOpMultiplicandBCongruous::Index
int32_t Index
Index type used for coordinates.
Definition: tensor_op_multiplicand_sm70.h:639
cutlass::layout::RowMajorVoltaTensorOpMultiplicandCongruous::stride
CUTLASS_HOST_DEVICE Stride stride() const
Returns the stride of the layout.
Definition: tensor_op_multiplicand_sm70.h:376
cutlass::layout::VoltaTensorOpMultiplicandCrosswise::LongIndex
int64_t LongIndex
Long index type used for offsets.
Definition: tensor_op_multiplicand_sm70.h:744
cutlass::layout::VoltaTensorOpMultiplicandCongruous::kStrideRank
static int const kStrideRank
Rank of stride vector.
Definition: tensor_op_multiplicand_sm70.h:66
cutlass::layout::RowMajorVoltaTensorOpMultiplicandCrosswise::LongIndex
int64_t LongIndex
Long index type used for offsets.
Definition: tensor_op_multiplicand_sm70.h:954
cutlass::layout::ColumnMajorVoltaTensorOpMultiplicandBCongruous::ColumnMajorVoltaTensorOpMultiplicandBCongruous
CUTLASS_HOST_DEVICE ColumnMajorVoltaTensorOpMultiplicandBCongruous(Stride stride)
Ctor.
Definition: tensor_op_multiplicand_sm70.h:587
cutlass::layout::ColumnMajorVoltaTensorOpMultiplicandCongruous::ColumnMajorVoltaTensorOpMultiplicandCongruous
CUTLASS_HOST_DEVICE ColumnMajorVoltaTensorOpMultiplicandCongruous(Index ldm=0)
Ctor.
Definition: tensor_op_multiplicand_sm70.h:246
cutlass::layout::VoltaTensorOpMultiplicandCongruous::VoltaTensorOpMultiplicandCongruous
CUTLASS_HOST_DEVICE VoltaTensorOpMultiplicandCongruous(Index ldm=0)
Ctor.
Definition: tensor_op_multiplicand_sm70.h:126
cutlass::layout::VoltaTensorOpMultiplicandCongruous::capacity
CUTLASS_HOST_DEVICE LongIndex capacity(TensorCoord const &extent) const
Compute the number of contiguous elements needed to store a tensor with the given size...
Definition: tensor_op_multiplicand_sm70.h:182
cutlass::layout::ColumnMajorVoltaTensorOpMultiplicandCongruous::AccessCount
typename Base::AccessCount AccessCount
Definition: tensor_op_multiplicand_sm70.h:229
cutlass::layout::VoltaTensorOpMultiplicandCrosswise::VoltaTensorOpMultiplicandCrosswise
CUTLASS_HOST_DEVICE VoltaTensorOpMultiplicandCrosswise(Index ldm=0)
Ctor.
Definition: tensor_op_multiplicand_sm70.h:781
cutlass::layout::VoltaTensorOpMultiplicandCongruous::kAccessSize
static int const kAccessSize
This layout is optimized for 128b accesses.
Definition: tensor_op_multiplicand_sm70.h:85
cutlass::layout::VoltaTensorOpMultiplicandCongruous::stride
CUTLASS_HOST_DEVICE Stride & stride()
Returns the stride of the layout.
Definition: tensor_op_multiplicand_sm70.h:176
pitch_linear.h
Defines layout functions used by TensorRef and derived classes for pitch-linear memory.
cutlass::layout::RowMajorVoltaTensorOpMultiplicandBCongruous::stride
CUTLASS_HOST_DEVICE Stride stride() const
Returns the stride of the layout.
Definition: tensor_op_multiplicand_sm70.h:713
cutlass::layout::VoltaTensorOpMultiplicandBCongruous::VoltaTensorOpMultiplicandBCongruous
CUTLASS_HOST_DEVICE VoltaTensorOpMultiplicandBCongruous(Index ldm=0)
Ctor.
Definition: tensor_op_multiplicand_sm70.h:462
cutlass::layout::ColumnMajorVoltaTensorOpMultiplicandCrosswise::Index
int32_t Index
Index type used for coordinates.
Definition: tensor_op_multiplicand_sm70.h:856
cutlass::layout::ColumnMajorVoltaTensorOpMultiplicandCongruous
Template mapping a column-major view of pitch-linear memory to VoltaTensorOpMultiplicandCongruous.
Definition: tensor_op_multiplicand_sm70.h:191
cutlass::layout::ColumnMajorVoltaTensorOpMultiplicandCongruous::capacity
CUTLASS_HOST_DEVICE LongIndex capacity(TensorCoord const &extent) const
Compute the number of contiguous elements needed to store a tensor with the given size...
Definition: tensor_op_multiplicand_sm70.h:286
cutlass::layout::RowMajorVoltaTensorOpMultiplicandBCongruous::operator()
CUTLASS_HOST_DEVICE LongIndex operator()(TensorCoord const &coord) const
Definition: tensor_op_multiplicand_sm70.h:700
cutlass::layout::RowMajorVoltaTensorOpMultiplicandCongruous::PartitionShape
typename Base::PartitionShape PartitionShape
Definition: tensor_op_multiplicand_sm70.h:322
cutlass::layout::ColumnMajorVoltaTensorOpMultiplicandCongruous::stride
CUTLASS_HOST_DEVICE Stride & stride()
Returns the stride of the layout.
Definition: tensor_op_multiplicand_sm70.h:280
cutlass::layout::ColumnMajorVoltaTensorOpMultiplicandCrosswise::ColumnMajorVoltaTensorOpMultiplicandCrosswise
CUTLASS_HOST_DEVICE ColumnMajorVoltaTensorOpMultiplicandCrosswise(Stride stride)
Ctor.
Definition: tensor_op_multiplicand_sm70.h:901
cutlass::layout::VoltaTensorOpMultiplicandBCongruous::LongIndex
int64_t LongIndex
Long index type used for offsets.
Definition: tensor_op_multiplicand_sm70.h:408
cutlass::layout::VoltaTensorOpMultiplicandCongruous
Template based on element size (in bits) - defined in terms of pitch-linear memory.
Definition: tensor_op_multiplicand_sm70.h:60
cutlass.h
Basic include for CUTLASS.
cutlass::MatrixCoord
Definition: matrix_coord.h:39
cutlass::layout::PitchLinearCoord::strided
CUTLASS_HOST_DEVICE Index const & strided() const
Returns the column of the coordinate.
Definition: pitch_linear.h:97
cutlass::layout::VoltaTensorOpMultiplicandCrosswise::stride
CUTLASS_HOST_DEVICE Stride stride() const
Returns the stride of the layout.
Definition: tensor_op_multiplicand_sm70.h:831
cutlass::layout::ColumnMajorVoltaTensorOpMultiplicandCongruous::ColumnMajorVoltaTensorOpMultiplicandCongruous
CUTLASS_HOST_DEVICE ColumnMajorVoltaTensorOpMultiplicandCongruous(Stride stride)
Ctor.
Definition: tensor_op_multiplicand_sm70.h:250
cutlass::layout::VoltaTensorOpMultiplicandCongruous::operator()
CUTLASS_HOST_DEVICE LongIndex operator()(TensorCoord const &coord) const
Definition: tensor_op_multiplicand_sm70.h:141
cutlass::layout::RowMajorVoltaTensorOpMultiplicandBCongruous::RowMajorVoltaTensorOpMultiplicandBCongruous
CUTLASS_HOST_DEVICE RowMajorVoltaTensorOpMultiplicandBCongruous(Index ldm=0)
Ctor.
Definition: tensor_op_multiplicand_sm70.h:685