opt PVC arch

Author: sunjiweiswift

examples/05_batch_gemm/batch_gemm.hpp

@@ -276,20 +276,20 @@ class batch_gemm_t {
                 args.matB_base.base, args.matB_ld);
       }
     }
-    if (epilogue_t::msg_type_c != msg_type::unaligned_2d) {
-      if (epilogue_t::msg_type_c == msg_type::block_2d) {
-        implementable &=
-            kernel::block_2d<gpu_arch::XeHpc, dtype_c>::check_tensor(
-                (uint64_t)(args.matC_base.base),
-                args.matrix_n,
-                args.matrix_m * args.batch_size,
-                args.matC_ld);
-      } else {
-        implementable &=
-            kernel::general_1d<gpu_arch::XeHpc, dtype_c>::check_alignment(
-                args.matC_base.base, args.matC_ld);
-      }
-    }
+    // if (epilogue_t::msg_type_c != msg_type::unaligned_2d) {
+    //   if (epilogue_t::msg_type_c == msg_type::block_2d) {
+    //     implementable &=
+    //         kernel::block_2d<gpu_arch::XeHpc, dtype_c>::check_tensor(
+    //             (uint64_t)(args.matC_base.base),
+    //             args.matrix_n,
+    //             args.matrix_m * args.batch_size,
+    //             args.matC_ld);
+    //   } else {
+    //     implementable &=
+    //         kernel::general_1d<gpu_arch::XeHpc, dtype_c>::check_alignment(
+    //             args.matC_base.base, args.matC_ld);
+    //   }
+    // }
 
     return implementable;
   }

examples/07_multi_layer_perceptron/multi_layer_perceptron.hpp

@@ -451,20 +451,20 @@ class multi_layer_perceptron_t {
                 args.matV_base.base, args.matV_ld);
       }
     }
-    if (epilogue_layer2_t::msg_type_c != msg_type::unaligned_2d) {
-      if (epilogue_layer2_t::msg_type_c == msg_type::block_2d) {
-        implementable &=
-            kernel::block_2d<gpu_arch::XeHpc, dtype_c>::check_tensor(
-                (uint64_t)(args.matC_base.base),
-                args.matrix_n_layer2,
-                args.matrix_m_layer2,
-                args.matC_ld);
-      } else {
-        implementable &=
-            kernel::general_1d<gpu_arch::XeHpc, dtype_c>::check_alignment(
-                args.matC_base.base, args.matC_ld);
-      }
-    }
+    // if (epilogue_layer2_t::msg_type_c != msg_type::unaligned_2d) {
+    //   if (epilogue_layer2_t::msg_type_c == msg_type::block_2d) {
+    //     implementable &=
+    //         kernel::block_2d<gpu_arch::XeHpc, dtype_c>::check_tensor(
+    //             (uint64_t)(args.matC_base.base),
+    //             args.matrix_n_layer2,
+    //             args.matrix_m_layer2,
+    //             args.matC_ld);
+    //   } else {
+    //     implementable &=
+    //         kernel::general_1d<gpu_arch::XeHpc, dtype_c>::check_alignment(
+    //             args.matC_base.base, args.matC_ld);
+    //   }
+    // }
 
     return implementable;
   }

include/common/core/arch_config.hpp

@@ -100,8 +100,8 @@ struct load_store_attr_t<msg_type::block_1d, arch_tag> {
 
 template <>
 struct load_store_attr_t<msg_type::block_1d, gpu_arch::XeHpc> {
-  static constexpr uint32_t max_load_vec_len = 64;
-  static constexpr uint32_t max_store_vec_len = 64;
+  static constexpr uint32_t max_load_vec_len = 512;
+  static constexpr uint32_t max_store_vec_len = 512;
   static constexpr uint32_t max_prefetch_vec_len = 64;
 };
 

include/common/core/memory.hpp

@@ -256,7 +256,7 @@ constexpr __ESIMD_NS::atomic_op get_atomic_op(gpu::xetla::atomic_op ao) {
 ///
 template <
     typename Ty,
-    uint8_t NElts = 1,
+    int NElts = 1,
     data_size DS = data_size::default_size,
     cache_hint L1H = cache_hint::cached,
     cache_hint L2H = cache_hint::cached,
@@ -293,7 +293,7 @@ __XETLA_API void xetla_prefetch_global(
 ///
 template <
     typename Ty,
-    uint8_t NElts = 1,
+    int NElts = 1,
     data_size DS = data_size::default_size,
     cache_hint L1H = cache_hint::cached,
     cache_hint L2H = cache_hint::cached>
@@ -385,7 +385,7 @@ __XETLA_API xetla_vector<T, N> xetla_load_global(
 ///
 template <
     typename Ty,
-    uint8_t NElts = 1,
+    int NElts = 1,
     data_size DS = data_size::default_size,
     cache_hint L1H = cache_hint::none,
     cache_hint L2H = cache_hint::none,
@@ -431,7 +431,7 @@ __XETLA_API xetla_vector<Ty, N * NElts> xetla_load_global(
 ///
 template <
     typename Ty,
-    uint8_t NElts = 1,
+    int NElts = 1,
     data_size DS = data_size::default_size,
     cache_hint L1H = cache_hint::none,
     cache_hint L2H = cache_hint::none,
@@ -653,7 +653,7 @@ __XETLA_API void xetla_local_init() {
 ///
 template <
     typename Ty,
-    uint8_t NElts = 1,
+    int NElts = 1,
     data_size DS = data_size::default_size,
     int N>
 __XETLA_API xetla_vector<Ty, N * NElts> xetla_load_local(
@@ -670,35 +670,31 @@ __XETLA_API xetla_vector<Ty, N * NElts> xetla_load_local(
           xetla_cvt<uint64_t, uint32_t>(offsets), pred);
 }
 
-/// @brief SLM block load. (transposed gather with 1 channel).
-/// Collects elements located at slm and returns them as a single \ref
-/// xetla_vector object.
-///
-/// Supported platforms: DG2, PVC
-///
-/// VISA instruction: lsc_load.slm
-///
-/// @tparam Ty is element type.
-/// @tparam NElts is the number of elements to load per address (i.e.
-/// vector_size per SIMD channel).
-/// @tparam DS is the data size.
-/// @param offset [in] is the zero-based offset for SLM buffer in bytes.
-/// @return is a xetla_vector of type T and size NElts.
-///
-template <
-    typename Ty,
-    uint8_t NElts = 1,
-    data_size DS = data_size::default_size>
+/// Loads a contiguous block of SLM memory referenced by the given byte-offset
+/// \p offset, then returns the loaded data as a simd object.
+/// The generated code depends on the combination {T, N, Flags}.
+/// Providing flags specifying the alignment of 16-bytes or more produces more
+/// efficient code. If the alignment is smaller than 16-bytes, then less
+/// efficient gather is generated. If the loaded vector is too long
+/// for 1 flat-load GPU instruction, then a series of flat-loads and/or gathers
+/// may be generated.
+/// @tparam T Element type.
+/// @tparam N Number of elements to load.
+/// @tparam Flags The alignment specifier type tag.
+/// @param byte_offset The byte-offset to load from.
+/// @param Flags Specifies the alignment.
+/// @return A vector of loaded elements.
+///
+template <typename Ty, int NElts = 1, data_size DS = data_size::default_size>
 __XETLA_API xetla_vector<Ty, NElts> xetla_load_local(uint32_t offset) {
   using T = native_type_t<Ty>;
-  DEBUG_INVOKE(
-      dbg_level::core,
-      core::general_1d<gpu_arch::XeHpc, Ty>::template check_restriction<NElts>(
-          (uint64_t)offset));
+  // DEBUG_INVOKE(
+  //     dbg_level::core,
+  //     core::general_1d<gpu_arch::XeHpc, Ty>::template
+  //     check_restriction<NElts>(
+  //         (uint64_t)offset));
 
-  return __ESIMD_ENS::
-      lsc_slm_block_load<T, NElts, gpu::xetla::detail::get_data_size(DS)>(
-          offset);
+  return __ESIMD_NS::slm_block_load<T, NElts>(offset);
 }
 
 /// @brief SLM scattered store.
@@ -719,7 +715,7 @@ __XETLA_API xetla_vector<Ty, NElts> xetla_load_local(uint32_t offset) {
 ///
 template <
     typename Ty,
-    uint8_t NElts = 1,
+    int NElts = 1,
     data_size DS = data_size::default_size,
     int N>
 __XETLA_API void xetla_store_local(
@@ -737,36 +733,38 @@ __XETLA_API void xetla_store_local(
           offsets, vals, pred);
 }
 
-/// @brief SLM block store (transposed SLM scatter with 1 channel).
-/// Scatters elements located to slm.
-///
-/// Supported platforms: DG2, PVC
-///
-/// VISA instruction: lsc_store.slm
-///
-/// @tparam Ty is element type.
-/// @tparam NElts is the number of elements to store per address (i.e.
-/// vector_size per SIMD channel).
-/// @tparam DS is the data size.
-/// @param offset [in] is the zero-based offset for SLM buffer in bytes.
-/// @param vals   [in] is values to store.
-///
-template <
-    typename Ty,
-    uint8_t NElts = 1,
-    data_size DS = data_size::default_size>
+/// Stores elements of the vector \p vals to a contiguous block of SLM memory
+/// at the given byte-offset \p offset.
+/// The generated code depends on the combination {T, N, Flags}.
+/// Providing flags specifying the alignment of 16-bytes or more produces more
+/// efficient code. If the alignment is smaller than 16-bytes, then less
+/// efficient scatter is generated. If the stored vector is too long
+/// for 1 flat-store GPU instruction, then a series of flat-store and/or
+/// scatters may be generated.
+/// @tparam T Element type.
+/// @tparam N Number of elements to store.
+/// @tparam Flags The alignment specifier type tag.
+/// @param offset The byte-offset to store at.
+/// @param vals The vector to store.
+/// @param Flags Specifies the alignment.
+///
+template <typename Ty, int NElts = 1, data_size DS = data_size::default_size>
 __XETLA_API void xetla_store_local(
     uint32_t offset,
     xetla_vector<Ty, NElts> vals) {
-  using T = native_type_t<Ty>;
-  DEBUG_INVOKE(
-      dbg_level::core,
-      core::general_1d<gpu_arch::XeHpc, Ty>::template check_restriction<NElts>(
-          offset));
-
-  __ESIMD_ENS::
-      lsc_slm_block_store<T, NElts, gpu::xetla::detail::get_data_size(DS)>(
-          offset, vals);
+  // using T = native_type_t<Ty>;
+  // DEBUG_INVOKE(
+  //     dbg_level::core,
+  //     core::general_1d<gpu_arch::XeHpc, Ty>::template
+  //     check_restriction<NElts>(
+  //         offset));
+
+  // __ESIMD_ENS::
+  //     lsc_slm_block_store<T, NElts, gpu::xetla::detail::get_data_size(DS)>(
+  //         offset, vals);
+  // __ESIMD_NS::properties props{};
+
+  __ESIMD_NS::slm_block_store<Ty, NElts>(offset, vals);
 }
 
 /// @brief SLM scattered atomic (0 src).

include/group/cooperative_reduction.hpp

@@ -95,7 +95,7 @@ class cooperative_reduce_t<
   static constexpr uint32_t block_size_x =
       gpu::xetla::subgroup::detail::gcd<tile_size_x, src_block_size_x>::value;
   static constexpr uint32_t block_size_y =
-      (tile_size_y > src_block_size_y) ? src_block_size_y : tile_size_y;
+      std::min(src_block_size_y, tile_size_y);
 
   using local_st_tile_desc_t = subgroup::tile_desc_t<
       sg_tile_n,

include/group/epilogue/impl/default_xe.hpp

@@ -70,9 +70,9 @@ class epilogue_t<
   }
 
  public:
-  static constexpr msg_type msg_type_c =
-      (mem_space_c == mem_space::global ? msg_type::block_2d
-                                        : msg_type::scatter);
+  //   static constexpr msg_type msg_type_c =
+  //       (mem_space_c == mem_space::global ? msg_type::block_2d
+  //                                         : msg_type::scatter);
 
   /// @brief Default epilogue.
   /// 1) Convert dtype_acc to dtype_c 2) Overwrite to memory.
@@ -94,6 +94,11 @@ class epilogue_t<
       [[maybe_unused]] uint32_t nbarrier_base = 0) {
     using mat_tile_desc = typename matAcc_t::tile_desc;
     using matC_t = subgroup::tile_t<dtype_c, mat_tile_desc>;
+
+    static constexpr msg_type msg_type_c =
+        subgroup::msg_type_v<mat_tile_desc, mem_desc_c_t>;
+    using matC_payload_t = subgroup::
+        mem_payload_t<mem_desc_c_t, mat_tile_desc, msg_type_c, arch_tag>;
     using matC_payload_t = subgroup::
         mem_payload_t<mem_desc_c_t, mat_tile_desc, msg_type_c, arch_tag>;
     update_sg_tile_tdesc(g, mem_desc_c);
@@ -143,9 +148,7 @@ class epilogue_t<
   using dtype_c = typename mem_desc_c_t::dtype;
   static constexpr mem_layout mem_layout_c = mem_desc_c_t::layout;
   static constexpr mem_space mem_space_c = mem_desc_c_t::space;
-  static constexpr msg_type msg_type_c =
-      (mem_space_c == mem_space::global ? msg_type::block_2d
-                                        : msg_type::scatter);
+
   /// @brief Updates tile base descriptor based on the tid.
   __XETLA_API static void update_sg_tile_tdesc(
       work_group_t& g,
@@ -165,8 +168,6 @@ class epilogue_t<
   }
 
  public:
-  static constexpr bool is_2d_block_c = (msg_type_c == msg_type::block_2d);
-
   /// @brief Default epilogue.
   /// 1) Convert dtype_acc to dtype_c 2) Overwrite to memory.
   /// @tparam matAcc_t Is the type of the input tile.
@@ -190,11 +191,13 @@ class epilogue_t<
       [[maybe_unused]] uint32_t nbarrier_base = 0) {
     using mat_tile_desc = typename matAcc_t::tile_desc;
     using matC_t = subgroup::tile_t<dtype_c, mat_tile_desc>;
-    using matC_payload_t = subgroup::mem_payload_t<
-        mem_desc_t<dtype_c, mem_layout_c, mem_space_c>,
-        mat_tile_desc,
-        msg_type_c,
-        arch_tag>;
+
+    // static constexpr msg_type msg_type_c = msg_type::block_2d;
+    static constexpr msg_type msg_type_c =
+        subgroup::msg_type_v<mat_tile_desc, mem_desc_c_t>;
+
+    using matC_payload_t = subgroup::
+        mem_payload_t<mem_desc_c_t, mat_tile_desc, msg_type_c, arch_tag>;
 
     update_sg_tile_tdesc(g, mem_desc_c);
 

include/group/epilogue/impl/tile_op_xe.hpp

@@ -127,6 +127,7 @@ class epilogue_t<
       uint32_t nbarrier_base = 0) {
     using mat_tile_desc = typename matAcc_t::tile_desc;
     using matC_t = subgroup::tile_t<dtype_c, mat_tile_desc>;
+    // static constexpr msg_type msg_type_c = msg_type::block_2d;
     static constexpr msg_type msg_type_c =
         subgroup::msg_type_v<mat_tile_desc, mem_desc_c_t>;
     using matC_payload_t = subgroup::

include/kernel/gemm/impl/kslicing_xe.hpp

@@ -387,18 +387,18 @@ class gemm_universal_t<
             args.matB_base.base, args.matB_ld);
       }
     }
-    if (epilogue_t::msg_type_c != msg_type::unaligned_2d) {
-      if (epilogue_t::msg_type_c == msg_type::block_2d) {
-        implementable &= kernel::block_2d<arch_tag, dtype_c>::check_tensor(
-            (uint64_t)(args.matC_base.base),
-            args.matrix_n,
-            args.matrix_m,
-            args.matC_ld);
-      } else {
-        implementable &= kernel::general_1d<arch_tag, dtype_c>::check_alignment(
-            args.matC_base.base, args.matC_ld);
-      }
-    }
+    // if (epilogue_t::msg_type_c != msg_type::unaligned_2d) {
+    //   if (epilogue_t::msg_type_c == msg_type::block_2d) {
+    //     implementable &= kernel::block_2d<arch_tag, dtype_c>::check_tensor(
+    //         (uint64_t)(args.matC_base.base),
+    //         args.matrix_n,
+    //         args.matrix_m,
+    //         args.matC_ld);
+    //   } else {
+    //     implementable &= kernel::general_1d<arch_tag, dtype_c>::check_alignment(
+    //         args.matC_base.base, args.matC_ld);
+    //   }
+    // }
 
     return implementable;
   }