modify rvalue reference for store_2d

Author: sunjiweiswift

include/common/core/memory.hpp

@@ -769,24 +769,24 @@ __XETLA_API void xetla_store_global(
     unsigned SurfacePitch,
     int X,
     int Y,
-    xetla_vector<T, N> Vals) {
+    auto&& Vals) {
   if constexpr (std::is_same_v<T, bf16>) {
-    xetla_vector<fp16, N> Vals_fp16 = Vals.xetla_format<fp16>();
     xetla_store_global<fp16, BlockWidth, BlockHeight, L1H, L2H>(
         reinterpret_cast<fp16*>(Ptr),
         SurfaceWidth,
         SurfaceHeight,
         SurfacePitch,
         X,
         Y,
-        Vals_fp16);
+        Vals.xetla_format<fp16>());
   } else {
     __ESIMD_ENS::lsc_store_2d<
         T,
         BlockWidth,
         BlockHeight,
         gpu::xetla::detail::get_cache_hint(L1H),
-        gpu::xetla::detail::get_cache_hint(L2H)>(
+        gpu::xetla::detail::get_cache_hint(L2H),
+        N>(
         Ptr, SurfaceWidth - 1, SurfaceHeight - 1, SurfacePitch - 1, X, Y, Vals);
   }
 }

include/subgroup/tile/impl/load_xe.hpp

@@ -89,7 +89,7 @@ tile_load(tile_t& tile, payload_t& payload) {
 
   static constexpr uint32_t num_block_x = tile_desc::num_block_x;
   static constexpr uint32_t num_block_y = tile_desc::num_block_y;
-//   static constexpr uint32_t num_block = tile_desc::num_block;
+  //   static constexpr uint32_t num_block = tile_desc::num_block;
 
   static constexpr gpu_arch arch_tag = payload_t::arch_tag;
 
@@ -329,7 +329,7 @@ tile_load(tile_t& tile, payload_t& payload) {
         reg_tmp.xetla_format<native_type_t<load_dtype>>() = xetla_load_global<
             native_type_t<load_dtype>,
             block_size_x / scale_factor,
-            block_size_y,
+            ld_blk_height,
             arr_len,
             trans,
             mem_transform,

include/subgroup/tile/impl/store_xe.hpp

@@ -98,7 +98,7 @@ tile_store(tile_t& tile, payload_t& payload) {
 
   static constexpr uint32_t num_block_x = tile_desc::num_block_x;
   static constexpr uint32_t num_block_y = tile_desc::num_block_y;
-//   static constexpr uint32_t num_block = tile_desc::num_block;
+  //   static constexpr uint32_t num_block = tile_desc::num_block;
 
   using load_store_attr = typename arch_attr_t<
       payload_t::arch_tag>::template load_store_attr<msg_type::block_2d>;
@@ -145,7 +145,7 @@ tile_store(tile_t& tile, payload_t& payload) {
 #pragma unroll
     for (uint32_t j = 0; j < num_block_x; j += arr_len) {
       int32_t offset_x = j * block_size_x;
-    //   xetla_tdescriptor tdesc = payload_row.row(j);
+      //   xetla_tdescriptor tdesc = payload_row.row(j);
       auto reg_blk = tile.reg.xetla_select<store_block_elems, 1>(
           (i * num_block_x + j) * block_elems);
       xetla_vector<dtype, store_block_elems> combine_blk;
@@ -163,7 +163,7 @@ tile_store(tile_t& tile, payload_t& payload) {
       for (uint32_t ii = 0; ii < block_size_y / st_block_size_y; ++ii) {
         constexpr uint32_t store_elems =
             st_block_size_y * block_size_x * arr_len;
-        xetla_vector<dtype, store_elems> st_blk =
+        auto st_blk =
             combine_blk.xetla_select<store_elems, 1>(ii * store_elems);
         // xetla_tstore_global<dtype, store_elems, L1, L2, payload_t::arch_tag>(
         //     tdesc, st_blk);
@@ -173,7 +173,7 @@ tile_store(tile_t& tile, payload_t& payload) {
             st_block_size_y,
             L1,
             L2>(
-            payload.base_ptr,
+            reinterpret_cast<dtype*>(payload.base_ptr),
             payload.surface_width,
             payload.surface_height,
             payload.surface_pitch,
@@ -210,7 +210,7 @@ tile_store(tile_t& tile, payload_t& payload) {
             blk_remained_y,
             L1,
             L2>(
-            payload.base_ptr,
+            reinterpret_cast<dtype*>(payload.base_ptr),
             payload.surface_width,
             payload.surface_height,
             payload.surface_pitch,
@@ -240,7 +240,7 @@ tile_store(tile_t& tile, payload_t& payload) {
 #pragma unroll
     for (uint32_t j = 0; j < num_block_x; j += arr_len) {
       int offset_x = j * block_size_x;
-    //   xetla_tdescriptor tdesc = payload_row.row(j);
+      //   xetla_tdescriptor tdesc = payload_row.row(j);
       auto reg_blk = tile.reg.xetla_select<remained_block_elems * arr_len, 1>(
           processed_elems + j * remained_block_elems);
       // Do combination
@@ -271,7 +271,7 @@ tile_store(tile_t& tile, payload_t& payload) {
             remained_st_blk_size_y,
             L1,
             L2>(
-            payload.base_ptr,
+            reinterpret_cast<dtype*>(payload.base_ptr),
             payload.surface_width,
             payload.surface_height,
             payload.surface_pitch,
@@ -308,7 +308,7 @@ tile_store(tile_t& tile, payload_t& payload) {
             final_st_blk_size_y,
             L1,
             L2>(
-            payload.base_ptr,
+            reinterpret_cast<dtype*>(payload.base_ptr),
             payload.surface_width,
             payload.surface_height,
             payload.surface_pitch,

tests/integration/default_config/group_gemm/kernel_func.hpp

@@ -108,6 +108,9 @@ struct default_config_group_gemm_test_func {
 
   using gemm_op_t = gemm_universal_t<dispatch_policy, gemm_t, epilogue_t>;
 
+  static constexpr uint32_t barrier_count = gemm_op_t::get_barrier_count();
+  static constexpr uint32_t slm_size = gemm_op_t::get_slm_size();
+
   static const char* func_name() {
     return "default_config_group_gemm_test_func";
   }

tests/integration/default_config/kernel_gemm/kernel_func.hpp

@@ -65,6 +65,9 @@ struct default_config_kernel_gemm_test_func {
       gpu_arch::XeHpc, // GPU arch
       tune_option>;
 
+  static constexpr uint32_t barrier_count = gemm_op_t::get_barrier_count();
+  static constexpr uint32_t slm_size = gemm_op_t::get_slm_size();
+
   static const char* func_name() {
     return "default_config_kernel_gemm_test_func";
   }

tests/integration/gemm/bf16/kernel_func.hpp

@@ -76,6 +76,9 @@ struct bf16_gemm_test_func {
 
   using gemm_op_t = gemm_universal_t<dispatch_policy, gemm_t, epilogue_t>;
 
+  static constexpr uint32_t barrier_count = gemm_op_t::get_barrier_count();
+  static constexpr uint32_t slm_size = gemm_op_t::get_slm_size();
+
   static const char* func_name() {
     return "bf16_gemm_test_func";
   }

tests/integration/gemm/fp32/kernel_func.hpp

@@ -77,6 +77,9 @@ struct fp32_gemm_test_func {
 
   using gemm_op_t = gemm_universal_t<dispatch_policy, gemm_t, epilogue_t>;
 
+  static constexpr uint32_t barrier_count = gemm_op_t::get_barrier_count();
+  static constexpr uint32_t slm_size = gemm_op_t::get_slm_size();
+
   static const char* func_name() {
     return "fp32_gemm_test_func";
   }

tests/integration/gemm/int8/kernel_func.hpp

@@ -72,6 +72,9 @@ struct int8gemm_test_func {
 
   using gemm_op_t = gemm_universal_t<dispatch_policy, gemm_t, epilogue_t>;
 
+  static constexpr uint32_t barrier_count = gemm_op_t::get_barrier_count();
+  static constexpr uint32_t slm_size = gemm_op_t::get_slm_size();
+
   static const char* func_name() {
     return "int8gemm_test_func";
   }

tests/integration/gemm/tf32/kernel_func.hpp

@@ -71,6 +71,9 @@ struct tf32_gemm_test_func {
 
   using gemm_op_t = gemm_universal_t<dispatch_policy, gemm_t, epilogue_t>;
 
+  static constexpr uint32_t barrier_count = gemm_op_t::get_barrier_count();
+  static constexpr uint32_t slm_size = gemm_op_t::get_slm_size();
+
   static const char* func_name() {
     return "tf32_gemm_test_func";
   }

tests/integration/gemm/unaligned_bf16/kernel_func.hpp

@@ -68,13 +68,20 @@ struct unaligned_gemm_test_func {
   using epilogue_t = epilogue_t<
       epilogue_policy_unaligned<arch_tag>,
       tile_shape,
-      mem_desc_t<dtype_c, mem_layout::row_major, mem_space::global, ldc_alignment>>;
+      mem_desc_t<
+          dtype_c,
+          mem_layout::row_major,
+          mem_space::global,
+          ldc_alignment>>;
 
   using group_swizzle = gpu::xetla::kernel::group_swizzle_default<arch_tag>;
   using dispatch_policy =
       dispatch_policy_kslicing<group_swizzle, global_kslicing, local_kslicing>;
   using gemm_op_t = gemm_universal_t<dispatch_policy, gemm_t, epilogue_t>;
 
+  static constexpr uint32_t barrier_count = gemm_op_t::get_barrier_count();
+  static constexpr uint32_t slm_size = gemm_op_t::get_slm_size();
+
   static const char* func_name() {
     return "unaligned_gemm_test_func";
   }