From 86350bbae4eca944670c1edef1fb25fa24d70801 Mon Sep 17 00:00:00 2001 From: hamlin Date: Tue, 20 May 2025 19:36:34 +0000 Subject: [PATCH 01/13] initial commit --- src/hotspot/share/opto/superword.cpp | 3 ++- src/hotspot/share/opto/vectornode.cpp | 4 ++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/src/hotspot/share/opto/superword.cpp b/src/hotspot/share/opto/superword.cpp index 5e34249aee810..741216ed1f0e0 100644 --- a/src/hotspot/share/opto/superword.cpp +++ b/src/hotspot/share/opto/superword.cpp @@ -2353,7 +2353,8 @@ bool SuperWord::is_velt_basic_type_compatible_use_def(Node* use, Node* def) cons } // Default case: input size of use equals output size of def. - return type2aelembytes(use_bt) == type2aelembytes(def_bt); + return (type2aelembytes(use_bt) == type2aelembytes(def_bt)) || + (use->is_CMove() && def->is_Bool()); } // Return nullptr if success, else failure message diff --git a/src/hotspot/share/opto/vectornode.cpp b/src/hotspot/share/opto/vectornode.cpp index 05ef64af70432..65eeb2214ffb2 100644 --- a/src/hotspot/share/opto/vectornode.cpp +++ b/src/hotspot/share/opto/vectornode.cpp @@ -86,6 +86,10 @@ int VectorNode::opcode(int sopc, BasicType bt) { return (bt == T_FLOAT ? Op_FmaVF : 0); case Op_FmaHF: return (bt == T_SHORT ? Op_FmaVHF : 0); + case Op_CMoveI: + return (bt == T_INT ? Op_VectorBlend : 0); + case Op_CMoveL: + return (bt == T_LONG ? Op_VectorBlend : 0); case Op_CMoveF: return (bt == T_FLOAT ? Op_VectorBlend : 0); case Op_CMoveD: From 58a7f7a37e1311860d19640fc2707aa1ee6fc989 Mon Sep 17 00:00:00 2001 From: hamlin Date: Wed, 21 May 2025 06:13:38 +0000 Subject: [PATCH 02/13] disable cmovei/l => vectorblend --- src/hotspot/share/opto/vectornode.cpp | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/hotspot/share/opto/vectornode.cpp b/src/hotspot/share/opto/vectornode.cpp index 65eeb2214ffb2..05ef64af70432 100644 --- a/src/hotspot/share/opto/vectornode.cpp +++ b/src/hotspot/share/opto/vectornode.cpp @@ -86,10 +86,6 @@ int VectorNode::opcode(int sopc, BasicType bt) { return (bt == T_FLOAT ? Op_FmaVF : 0); case Op_FmaHF: return (bt == T_SHORT ? Op_FmaVHF : 0); - case Op_CMoveI: - return (bt == T_INT ? Op_VectorBlend : 0); - case Op_CMoveL: - return (bt == T_LONG ? Op_VectorBlend : 0); case Op_CMoveF: return (bt == T_FLOAT ? Op_VectorBlend : 0); case Op_CMoveD: From e967feac194d82b1bb8fb5a1e9814bc9bbed4875 Mon Sep 17 00:00:00 2001 From: hamlin Date: Thu, 22 May 2025 20:11:49 +0000 Subject: [PATCH 03/13] split from pr 25341 --- src/hotspot/cpu/aarch64/matcher_aarch64.hpp | 15 +++++++++++++++ src/hotspot/cpu/arm/matcher_arm.hpp | 14 ++++++++++++++ src/hotspot/cpu/ppc/matcher_ppc.hpp | 14 ++++++++++++++ src/hotspot/cpu/riscv/matcher_riscv.hpp | 14 ++++++++++++++ src/hotspot/cpu/s390/matcher_s390.hpp | 14 ++++++++++++++ src/hotspot/cpu/x86/matcher_x86.hpp | 14 ++++++++++++++ src/hotspot/share/opto/superword.cpp | 14 +++++++++++--- src/hotspot/share/opto/vectornode.cpp | 19 +++++++++++++++++++ src/hotspot/share/opto/vectornode.hpp | 5 +++++ 9 files changed, 120 insertions(+), 3 deletions(-) diff --git a/src/hotspot/cpu/aarch64/matcher_aarch64.hpp b/src/hotspot/cpu/aarch64/matcher_aarch64.hpp index 0fbc2ef141e8b..58f0faf103ac0 100644 --- a/src/hotspot/cpu/aarch64/matcher_aarch64.hpp +++ b/src/hotspot/cpu/aarch64/matcher_aarch64.hpp @@ -204,4 +204,19 @@ static bool is_feat_fp16_supported() { return (VM_Version::supports_fphp() && VM_Version::supports_asimdhp()); } + + static bool supports_vectorize_cmove_bool_unconditionally() { + return false; + } + + static bool supports_transform_cmove_to_vectorblend(int cmove_opc) { + switch (cmove_opc) { + case Op_CMoveF: + case Op_CMoveD: + return true; + default: + return false; + } + } + #endif // CPU_AARCH64_MATCHER_AARCH64_HPP diff --git a/src/hotspot/cpu/arm/matcher_arm.hpp b/src/hotspot/cpu/arm/matcher_arm.hpp index 66fe8ac330eb5..fa5b54358da68 100644 --- a/src/hotspot/cpu/arm/matcher_arm.hpp +++ b/src/hotspot/cpu/arm/matcher_arm.hpp @@ -193,4 +193,18 @@ return false; } + static bool supports_vectorize_cmove_bool_unconditionally() { + return false; + } + + static bool supports_transform_cmove_to_vectorblend(int cmove_opc) { + switch (cmove_opc) { + case Op_CMoveF: + case Op_CMoveD: + return true; + default: + return false; + } + } + #endif // CPU_ARM_MATCHER_ARM_HPP diff --git a/src/hotspot/cpu/ppc/matcher_ppc.hpp b/src/hotspot/cpu/ppc/matcher_ppc.hpp index 666bec9e0c802..03d618165db3e 100644 --- a/src/hotspot/cpu/ppc/matcher_ppc.hpp +++ b/src/hotspot/cpu/ppc/matcher_ppc.hpp @@ -204,4 +204,18 @@ return false; } + static bool supports_vectorize_cmove_bool_unconditionally() { + return false; + } + + static bool supports_transform_cmove_to_vectorblend(int cmove_opc) { + switch (cmove_opc) { + case Op_CMoveF: + case Op_CMoveD: + return true; + default: + return false; + } + } + #endif // CPU_PPC_MATCHER_PPC_HPP diff --git a/src/hotspot/cpu/riscv/matcher_riscv.hpp b/src/hotspot/cpu/riscv/matcher_riscv.hpp index 1b490a07f92a6..3e9af4d3d443f 100644 --- a/src/hotspot/cpu/riscv/matcher_riscv.hpp +++ b/src/hotspot/cpu/riscv/matcher_riscv.hpp @@ -199,4 +199,18 @@ return false; } + static bool supports_vectorize_cmove_bool_unconditionally() { + return true; + } + + static bool supports_transform_cmove_to_vectorblend(int cmove_opc) { + switch (cmove_opc) { + case Op_CMoveF: + case Op_CMoveD: + return true; + default: + return false; + } + } + #endif // CPU_RISCV_MATCHER_RISCV_HPP diff --git a/src/hotspot/cpu/s390/matcher_s390.hpp b/src/hotspot/cpu/s390/matcher_s390.hpp index e4c277c63a8b9..fc118d2c0b790 100644 --- a/src/hotspot/cpu/s390/matcher_s390.hpp +++ b/src/hotspot/cpu/s390/matcher_s390.hpp @@ -196,4 +196,18 @@ return false; } + static bool supports_vectorize_cmove_bool_unconditionally() { + return false; + } + + static bool supports_transform_cmove_to_vectorblend(int cmove_opc) { + switch (cmove_opc) { + case Op_CMoveF: + case Op_CMoveD: + return true; + default: + return false; + } + } + #endif // CPU_S390_MATCHER_S390_HPP diff --git a/src/hotspot/cpu/x86/matcher_x86.hpp b/src/hotspot/cpu/x86/matcher_x86.hpp index 41486c244b247..b7453b5f27529 100644 --- a/src/hotspot/cpu/x86/matcher_x86.hpp +++ b/src/hotspot/cpu/x86/matcher_x86.hpp @@ -236,4 +236,18 @@ } } + static bool supports_vectorize_cmove_bool_unconditionally() { + return false; + } + + static bool supports_transform_cmove_to_vectorblend(int cmove_opc) { + switch (cmove_opc) { + case Op_CMoveF: + case Op_CMoveD: + return true; + default: + return false; + } + } + #endif // CPU_X86_MATCHER_X86_HPP diff --git a/src/hotspot/share/opto/superword.cpp b/src/hotspot/share/opto/superword.cpp index 741216ed1f0e0..fc08b7fcfcde1 100644 --- a/src/hotspot/share/opto/superword.cpp +++ b/src/hotspot/share/opto/superword.cpp @@ -2352,9 +2352,17 @@ bool SuperWord::is_velt_basic_type_compatible_use_def(Node* use, Node* def) cons type2aelembytes(use_bt) == 4; } - // Default case: input size of use equals output size of def. - return (type2aelembytes(use_bt) == type2aelembytes(def_bt)) || - (use->is_CMove() && def->is_Bool()); + // Input size of use equals output size of def + if (type2aelembytes(use_bt) == type2aelembytes(def_bt)) { + return true; + } + + if (use->is_CMove() && def->is_Bool() && + VectorNode::is_vectorize_cmove_bool_unconditionally_supported()) { + return true; + } + + return false; } // Return nullptr if success, else failure message diff --git a/src/hotspot/share/opto/vectornode.cpp b/src/hotspot/share/opto/vectornode.cpp index 05ef64af70432..2976efec7c7a9 100644 --- a/src/hotspot/share/opto/vectornode.cpp +++ b/src/hotspot/share/opto/vectornode.cpp @@ -86,6 +86,10 @@ int VectorNode::opcode(int sopc, BasicType bt) { return (bt == T_FLOAT ? Op_FmaVF : 0); case Op_FmaHF: return (bt == T_SHORT ? Op_FmaVHF : 0); + case Op_CMoveI: + return (bt == T_INT ? Op_VectorBlend : 0); + case Op_CMoveL: + return (bt == T_LONG ? Op_VectorBlend : 0); case Op_CMoveF: return (bt == T_FLOAT ? Op_VectorBlend : 0); case Op_CMoveD: @@ -429,6 +433,9 @@ bool VectorNode::implemented(int opc, uint vlen, BasicType bt) { if (VectorNode::is_vector_integral_negate(vopc)) { return is_vector_integral_negate_supported(vopc, vlen, bt, false); } + if (vopc == Op_VectorBlend) { + return VectorBlendNode::implemented(opc); + } return vopc > 0 && Matcher::match_rule_supported_auto_vectorization(vopc, vlen, bt); } return false; @@ -511,6 +518,10 @@ bool VectorNode::is_populate_index_supported(BasicType bt) { return Matcher::match_rule_supported_vector(Op_PopulateIndex, vlen, bt); } +bool VectorNode::is_vectorize_cmove_bool_unconditionally_supported() { + return Matcher::supports_vectorize_cmove_bool_unconditionally(); +} + bool VectorNode::is_shift_opcode(int opc) { switch (opc) { case Op_LShiftI: @@ -2187,6 +2198,14 @@ Node* VectorBlendNode::Identity(PhaseGVN* phase) { } return this; } + +bool VectorBlendNode::implemented(int opc) { + assert(opc == Op_CMoveF || opc == Op_CMoveD || + opc == Op_CMoveI || opc == Op_CMoveL || + opc == Op_CMoveN || opc == Op_CMoveP, "must"); + return Matcher::supports_transform_cmove_to_vectorblend(opc); +} + static bool is_replicate_uint_constant(const Node* n) { return n->Opcode() == Op_Replicate && n->in(1)->is_Con() && diff --git a/src/hotspot/share/opto/vectornode.hpp b/src/hotspot/share/opto/vectornode.hpp index e72c3880c7983..7fd6863b54338 100644 --- a/src/hotspot/share/opto/vectornode.hpp +++ b/src/hotspot/share/opto/vectornode.hpp @@ -110,6 +110,9 @@ class VectorNode : public TypeNode { static bool is_vector_rotate_supported(int opc, uint vlen, BasicType bt); static bool is_vector_integral_negate_supported(int opc, uint vlen, BasicType bt, bool use_predicate); static bool is_populate_index_supported(BasicType bt); + // Supports to vectorize CMove which uses Bool unconditionally? + // Otherwise only same type size of CMove and Bool will be supported for this transformation. + static bool is_vectorize_cmove_bool_unconditionally_supported(); // Return true if every bit in this vector is 1. static bool is_all_ones_vector(Node* n); // Return true if every bit in this vector is 0. @@ -1736,6 +1739,8 @@ class VectorBlendNode : public VectorNode { Node* vec1() const { return in(1); } Node* vec2() const { return in(2); } Node* vec_mask() const { return in(3); } + + static bool implemented(int opc); }; class VectorRearrangeNode : public VectorNode { From e27247ba8fb95ab7b4872c63fb684cb95344d573 Mon Sep 17 00:00:00 2001 From: hamlin Date: Fri, 12 Sep 2025 09:04:32 +0000 Subject: [PATCH 04/13] initial commit --- src/hotspot/cpu/riscv/nativeInst_riscv.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/hotspot/cpu/riscv/nativeInst_riscv.cpp b/src/hotspot/cpu/riscv/nativeInst_riscv.cpp index 9722fd49d4d3a..72cc95a595dbf 100644 --- a/src/hotspot/cpu/riscv/nativeInst_riscv.cpp +++ b/src/hotspot/cpu/riscv/nativeInst_riscv.cpp @@ -28,6 +28,7 @@ #include "code/compiledIC.hpp" #include "nativeInst_riscv.hpp" #include "oops/oop.inline.hpp" +#include "runtime/atomicAccess.hpp" #include "runtime/handles.hpp" #include "runtime/orderAccess.hpp" #include "runtime/safepoint.hpp" @@ -99,10 +100,10 @@ void NativeCall::optimize_call(address dest, bool mt_safe) { if (Assembler::reachable_from_branch_at(jmp_ins_pc, dest)) { int64_t distance = dest - jmp_ins_pc; uint32_t new_jal = Assembler::encode_jal(ra, distance); - Atomic::store((uint32_t *)jmp_ins_pc, new_jal); + AtomicAccess::store((uint32_t *)jmp_ins_pc, new_jal); } else if (!MacroAssembler::is_jalr_at(jmp_ins_pc)) { // The jalr is always identical: jalr ra, 0(t1) uint32_t new_jalr = Assembler::encode_jalr(ra, t1, 0); - Atomic::store((uint32_t *)jmp_ins_pc, new_jalr); + AtomicAccess::store((uint32_t *)jmp_ins_pc, new_jalr); } else { // No change to instruction stream return; From 2ba466b7b52a9e1ab97042ae399ffa18546328cd Mon Sep 17 00:00:00 2001 From: hamlin Date: Tue, 4 Nov 2025 20:08:54 +0000 Subject: [PATCH 05/13] disable riscv --- src/hotspot/cpu/riscv/matcher_riscv.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/hotspot/cpu/riscv/matcher_riscv.hpp b/src/hotspot/cpu/riscv/matcher_riscv.hpp index 3e9af4d3d443f..d959324365c33 100644 --- a/src/hotspot/cpu/riscv/matcher_riscv.hpp +++ b/src/hotspot/cpu/riscv/matcher_riscv.hpp @@ -200,7 +200,7 @@ } static bool supports_vectorize_cmove_bool_unconditionally() { - return true; + return false; } static bool supports_transform_cmove_to_vectorblend(int cmove_opc) { From 2a0e1ad82091a8046f6e13c2d27a271f46a68461 Mon Sep 17 00:00:00 2001 From: hamlin Date: Tue, 4 Nov 2025 20:24:15 +0000 Subject: [PATCH 06/13] disable Op_CMoveI/Op_CMoveL in VectorNode::opcode --- src/hotspot/share/opto/vectornode.cpp | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/hotspot/share/opto/vectornode.cpp b/src/hotspot/share/opto/vectornode.cpp index 0ece1db288334..8c0b0b3c288fe 100644 --- a/src/hotspot/share/opto/vectornode.cpp +++ b/src/hotspot/share/opto/vectornode.cpp @@ -86,10 +86,6 @@ int VectorNode::opcode(int sopc, BasicType bt) { return (bt == T_FLOAT ? Op_FmaVF : 0); case Op_FmaHF: return (bt == T_SHORT ? Op_FmaVHF : 0); - case Op_CMoveI: - return (bt == T_INT ? Op_VectorBlend : 0); - case Op_CMoveL: - return (bt == T_LONG ? Op_VectorBlend : 0); case Op_CMoveF: return (bt == T_FLOAT ? Op_VectorBlend : 0); case Op_CMoveD: From 9e5f137fd83b86b710ea0af28f66d6c35db108fa Mon Sep 17 00:00:00 2001 From: hamlin Date: Tue, 4 Nov 2025 20:59:06 +0000 Subject: [PATCH 07/13] revert supports_transform_cmove_to_vectorblend for all cpus --- src/hotspot/cpu/aarch64/matcher_aarch64.hpp | 8 +------- src/hotspot/cpu/arm/matcher_arm.hpp | 8 +------- src/hotspot/cpu/ppc/matcher_ppc.hpp | 8 +------- src/hotspot/cpu/riscv/matcher_riscv.hpp | 8 +------- src/hotspot/cpu/s390/matcher_s390.hpp | 8 +------- src/hotspot/cpu/x86/matcher_x86.hpp | 8 +------- 6 files changed, 6 insertions(+), 42 deletions(-) diff --git a/src/hotspot/cpu/aarch64/matcher_aarch64.hpp b/src/hotspot/cpu/aarch64/matcher_aarch64.hpp index 58f0faf103ac0..78efebbae1f23 100644 --- a/src/hotspot/cpu/aarch64/matcher_aarch64.hpp +++ b/src/hotspot/cpu/aarch64/matcher_aarch64.hpp @@ -210,13 +210,7 @@ } static bool supports_transform_cmove_to_vectorblend(int cmove_opc) { - switch (cmove_opc) { - case Op_CMoveF: - case Op_CMoveD: - return true; - default: - return false; - } + return false; } #endif // CPU_AARCH64_MATCHER_AARCH64_HPP diff --git a/src/hotspot/cpu/arm/matcher_arm.hpp b/src/hotspot/cpu/arm/matcher_arm.hpp index fa5b54358da68..442bb0a40070f 100644 --- a/src/hotspot/cpu/arm/matcher_arm.hpp +++ b/src/hotspot/cpu/arm/matcher_arm.hpp @@ -198,13 +198,7 @@ } static bool supports_transform_cmove_to_vectorblend(int cmove_opc) { - switch (cmove_opc) { - case Op_CMoveF: - case Op_CMoveD: - return true; - default: - return false; - } + return false; } #endif // CPU_ARM_MATCHER_ARM_HPP diff --git a/src/hotspot/cpu/ppc/matcher_ppc.hpp b/src/hotspot/cpu/ppc/matcher_ppc.hpp index 908d2d172ba80..91c4825facccf 100644 --- a/src/hotspot/cpu/ppc/matcher_ppc.hpp +++ b/src/hotspot/cpu/ppc/matcher_ppc.hpp @@ -208,13 +208,7 @@ } static bool supports_transform_cmove_to_vectorblend(int cmove_opc) { - switch (cmove_opc) { - case Op_CMoveF: - case Op_CMoveD: - return true; - default: - return false; - } + return false; } #endif // CPU_PPC_MATCHER_PPC_HPP diff --git a/src/hotspot/cpu/riscv/matcher_riscv.hpp b/src/hotspot/cpu/riscv/matcher_riscv.hpp index d959324365c33..8499182e7f1dd 100644 --- a/src/hotspot/cpu/riscv/matcher_riscv.hpp +++ b/src/hotspot/cpu/riscv/matcher_riscv.hpp @@ -204,13 +204,7 @@ } static bool supports_transform_cmove_to_vectorblend(int cmove_opc) { - switch (cmove_opc) { - case Op_CMoveF: - case Op_CMoveD: - return true; - default: - return false; - } + return false; } #endif // CPU_RISCV_MATCHER_RISCV_HPP diff --git a/src/hotspot/cpu/s390/matcher_s390.hpp b/src/hotspot/cpu/s390/matcher_s390.hpp index fc118d2c0b790..bdadfa1929286 100644 --- a/src/hotspot/cpu/s390/matcher_s390.hpp +++ b/src/hotspot/cpu/s390/matcher_s390.hpp @@ -201,13 +201,7 @@ } static bool supports_transform_cmove_to_vectorblend(int cmove_opc) { - switch (cmove_opc) { - case Op_CMoveF: - case Op_CMoveD: - return true; - default: - return false; - } + return false; } #endif // CPU_S390_MATCHER_S390_HPP diff --git a/src/hotspot/cpu/x86/matcher_x86.hpp b/src/hotspot/cpu/x86/matcher_x86.hpp index b7453b5f27529..580515434c857 100644 --- a/src/hotspot/cpu/x86/matcher_x86.hpp +++ b/src/hotspot/cpu/x86/matcher_x86.hpp @@ -241,13 +241,7 @@ } static bool supports_transform_cmove_to_vectorblend(int cmove_opc) { - switch (cmove_opc) { - case Op_CMoveF: - case Op_CMoveD: - return true; - default: - return false; - } + return false; } #endif // CPU_X86_MATCHER_X86_HPP From bc0c9b3fa2af8c21b51545f90ee58720f413dcb9 Mon Sep 17 00:00:00 2001 From: hamlin Date: Tue, 4 Nov 2025 22:23:02 +0000 Subject: [PATCH 08/13] fix JDK-8371297: assert in BoolTest --- src/hotspot/share/opto/vtransform.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/hotspot/share/opto/vtransform.cpp b/src/hotspot/share/opto/vtransform.cpp index 46e8f43cb657d..40c5206bccb95 100644 --- a/src/hotspot/share/opto/vtransform.cpp +++ b/src/hotspot/share/opto/vtransform.cpp @@ -1421,8 +1421,9 @@ void VTransformReinterpretVectorNode::print_spec() const { void VTransformBoolVectorNode::print_spec() const { VTransformVectorNode::print_spec(); - const BoolTest bt(_test._mask); - tty->print(" test="); + BoolTest::mask m = BoolTest::mask(_test._mask & ~BoolTest::unsigned_compare); + const BoolTest bt(m); + tty->print(" test=%s", m == _test._mask ? "" : "unsigned "); bt.dump_on(tty); } #endif From 5b85c740a13daa24b66b6b6e621d82cdeac633a9 Mon Sep 17 00:00:00 2001 From: hamlin Date: Wed, 5 Nov 2025 09:32:15 +0000 Subject: [PATCH 09/13] fix code path change in VectorNode::implemented --- src/hotspot/share/opto/vectornode.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/hotspot/share/opto/vectornode.cpp b/src/hotspot/share/opto/vectornode.cpp index 8c0b0b3c288fe..299d3f5b5b621 100644 --- a/src/hotspot/share/opto/vectornode.cpp +++ b/src/hotspot/share/opto/vectornode.cpp @@ -314,8 +314,8 @@ bool VectorNode::implemented(int opc, uint vlen, BasicType bt) { if (VectorNode::is_vector_integral_negate(vopc)) { return is_vector_integral_negate_supported(vopc, vlen, bt, false); } - if (vopc == Op_VectorBlend) { - return VectorBlendNode::implemented(opc); + if (vopc == Op_VectorBlend && VectorBlendNode::implemented(opc)) { + return true; } return vopc > 0 && Matcher::match_rule_supported_auto_vectorization(vopc, vlen, bt); } From 81996cfe126ef5e160800c406c292ddb0b357905 Mon Sep 17 00:00:00 2001 From: hamlin Date: Wed, 5 Nov 2025 11:55:21 +0000 Subject: [PATCH 10/13] simplify --- src/hotspot/cpu/aarch64/matcher_aarch64.hpp | 6 +----- src/hotspot/cpu/arm/matcher_arm.hpp | 6 +----- src/hotspot/cpu/ppc/matcher_ppc.hpp | 6 +----- src/hotspot/cpu/riscv/matcher_riscv.hpp | 6 +----- src/hotspot/cpu/s390/matcher_s390.hpp | 6 +----- src/hotspot/cpu/x86/matcher_x86.hpp | 6 +----- src/hotspot/share/opto/superword.cpp | 7 ++----- src/hotspot/share/opto/vectornode.cpp | 14 ++------------ src/hotspot/share/opto/vectornode.hpp | 4 +--- 9 files changed, 11 insertions(+), 50 deletions(-) diff --git a/src/hotspot/cpu/aarch64/matcher_aarch64.hpp b/src/hotspot/cpu/aarch64/matcher_aarch64.hpp index 78efebbae1f23..49eae1cc438a1 100644 --- a/src/hotspot/cpu/aarch64/matcher_aarch64.hpp +++ b/src/hotspot/cpu/aarch64/matcher_aarch64.hpp @@ -205,11 +205,7 @@ return (VM_Version::supports_fphp() && VM_Version::supports_asimdhp()); } - static bool supports_vectorize_cmove_bool_unconditionally() { - return false; - } - - static bool supports_transform_cmove_to_vectorblend(int cmove_opc) { + static bool supports_vector_different_use_def_size() { return false; } diff --git a/src/hotspot/cpu/arm/matcher_arm.hpp b/src/hotspot/cpu/arm/matcher_arm.hpp index 442bb0a40070f..28e9caf1ab084 100644 --- a/src/hotspot/cpu/arm/matcher_arm.hpp +++ b/src/hotspot/cpu/arm/matcher_arm.hpp @@ -193,11 +193,7 @@ return false; } - static bool supports_vectorize_cmove_bool_unconditionally() { - return false; - } - - static bool supports_transform_cmove_to_vectorblend(int cmove_opc) { + static bool supports_vector_different_use_def_size() { return false; } diff --git a/src/hotspot/cpu/ppc/matcher_ppc.hpp b/src/hotspot/cpu/ppc/matcher_ppc.hpp index 91c4825facccf..987b959aa3b43 100644 --- a/src/hotspot/cpu/ppc/matcher_ppc.hpp +++ b/src/hotspot/cpu/ppc/matcher_ppc.hpp @@ -203,11 +203,7 @@ return false; } - static bool supports_vectorize_cmove_bool_unconditionally() { - return false; - } - - static bool supports_transform_cmove_to_vectorblend(int cmove_opc) { + static bool supports_vector_different_use_def_size() { return false; } diff --git a/src/hotspot/cpu/riscv/matcher_riscv.hpp b/src/hotspot/cpu/riscv/matcher_riscv.hpp index 8499182e7f1dd..f3084758b28e8 100644 --- a/src/hotspot/cpu/riscv/matcher_riscv.hpp +++ b/src/hotspot/cpu/riscv/matcher_riscv.hpp @@ -199,11 +199,7 @@ return false; } - static bool supports_vectorize_cmove_bool_unconditionally() { - return false; - } - - static bool supports_transform_cmove_to_vectorblend(int cmove_opc) { + static bool supports_vector_different_use_def_size() { return false; } diff --git a/src/hotspot/cpu/s390/matcher_s390.hpp b/src/hotspot/cpu/s390/matcher_s390.hpp index bdadfa1929286..8ab5587aaba20 100644 --- a/src/hotspot/cpu/s390/matcher_s390.hpp +++ b/src/hotspot/cpu/s390/matcher_s390.hpp @@ -196,11 +196,7 @@ return false; } - static bool supports_vectorize_cmove_bool_unconditionally() { - return false; - } - - static bool supports_transform_cmove_to_vectorblend(int cmove_opc) { + static bool supports_vector_different_use_def_size() { return false; } diff --git a/src/hotspot/cpu/x86/matcher_x86.hpp b/src/hotspot/cpu/x86/matcher_x86.hpp index 580515434c857..d12cb7cb84276 100644 --- a/src/hotspot/cpu/x86/matcher_x86.hpp +++ b/src/hotspot/cpu/x86/matcher_x86.hpp @@ -236,11 +236,7 @@ } } - static bool supports_vectorize_cmove_bool_unconditionally() { - return false; - } - - static bool supports_transform_cmove_to_vectorblend(int cmove_opc) { + static bool supports_vector_different_use_def_size() { return false; } diff --git a/src/hotspot/share/opto/superword.cpp b/src/hotspot/share/opto/superword.cpp index c8e4e43b457d9..dea8be591d7ac 100644 --- a/src/hotspot/share/opto/superword.cpp +++ b/src/hotspot/share/opto/superword.cpp @@ -2334,11 +2334,8 @@ bool SuperWord::is_velt_basic_type_compatible_use_def(Node* use, Node* def) cons // Input size of use equals output size of def if (type2aelembytes(use_bt) == type2aelembytes(def_bt)) { return true; - } - - if (use->is_CMove() && def->is_Bool() && - VectorNode::is_vectorize_cmove_bool_unconditionally_supported()) { - return true; + } else if (VectorNode::is_different_use_def_size_supported()) { + return use->is_CMove() && def->is_Bool(); } return false; diff --git a/src/hotspot/share/opto/vectornode.cpp b/src/hotspot/share/opto/vectornode.cpp index 299d3f5b5b621..b6480c3a9652a 100644 --- a/src/hotspot/share/opto/vectornode.cpp +++ b/src/hotspot/share/opto/vectornode.cpp @@ -314,9 +314,6 @@ bool VectorNode::implemented(int opc, uint vlen, BasicType bt) { if (VectorNode::is_vector_integral_negate(vopc)) { return is_vector_integral_negate_supported(vopc, vlen, bt, false); } - if (vopc == Op_VectorBlend && VectorBlendNode::implemented(opc)) { - return true; - } return vopc > 0 && Matcher::match_rule_supported_auto_vectorization(vopc, vlen, bt); } return false; @@ -409,8 +406,8 @@ bool VectorNode::is_populate_index_supported(BasicType bt) { return Matcher::match_rule_supported_vector(Op_PopulateIndex, vlen, bt); } -bool VectorNode::is_vectorize_cmove_bool_unconditionally_supported() { - return Matcher::supports_vectorize_cmove_bool_unconditionally(); +bool VectorNode::is_different_use_def_size_supported() { + return Matcher::supports_vector_different_use_def_size(); } bool VectorNode::is_shift_opcode(int opc) { @@ -2310,13 +2307,6 @@ Node* VectorBlendNode::Identity(PhaseGVN* phase) { return this; } -bool VectorBlendNode::implemented(int opc) { - assert(opc == Op_CMoveF || opc == Op_CMoveD || - opc == Op_CMoveI || opc == Op_CMoveL || - opc == Op_CMoveN || opc == Op_CMoveP, "must"); - return Matcher::supports_transform_cmove_to_vectorblend(opc); -} - static bool is_replicate_uint_constant(const Node* n) { return n->Opcode() == Op_Replicate && n->in(1)->is_Con() && diff --git a/src/hotspot/share/opto/vectornode.hpp b/src/hotspot/share/opto/vectornode.hpp index 330fe17e88d05..915e114cb252c 100644 --- a/src/hotspot/share/opto/vectornode.hpp +++ b/src/hotspot/share/opto/vectornode.hpp @@ -112,7 +112,7 @@ class VectorNode : public TypeNode { static bool is_populate_index_supported(BasicType bt); // Supports to vectorize CMove which uses Bool unconditionally? // Otherwise only same type size of CMove and Bool will be supported for this transformation. - static bool is_vectorize_cmove_bool_unconditionally_supported(); + static bool is_different_use_def_size_supported(); // Return true if every bit in this vector is 1. static bool is_all_ones_vector(Node* n); // Return true if every bit in this vector is 0. @@ -1734,8 +1734,6 @@ class VectorBlendNode : public VectorNode { Node* vec1() const { return in(1); } Node* vec2() const { return in(2); } Node* vec_mask() const { return in(3); } - - static bool implemented(int opc); }; class VectorRearrangeNode : public VectorNode { From 56b6e02938103436dc92001c9813212b5874b4ba Mon Sep 17 00:00:00 2001 From: hamlin Date: Wed, 5 Nov 2025 13:41:04 +0000 Subject: [PATCH 11/13] comments --- src/hotspot/share/opto/vectornode.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/hotspot/share/opto/vectornode.hpp b/src/hotspot/share/opto/vectornode.hpp index 915e114cb252c..aba825081474e 100644 --- a/src/hotspot/share/opto/vectornode.hpp +++ b/src/hotspot/share/opto/vectornode.hpp @@ -110,8 +110,8 @@ class VectorNode : public TypeNode { static bool is_vector_rotate_supported(int opc, uint vlen, BasicType bt); static bool is_vector_integral_negate_supported(int opc, uint vlen, BasicType bt, bool use_predicate); static bool is_populate_index_supported(BasicType bt); - // Supports to vectorize CMove which uses Bool unconditionally? - // Otherwise only same type size of CMove and Bool will be supported for this transformation. + // Return true if every bit in this vector is 1, e.g. based on the comparison + // result of 2 floats, set a double result. static bool is_different_use_def_size_supported(); // Return true if every bit in this vector is 1. static bool is_all_ones_vector(Node* n); From cfbe0a655df61efa89336895dd315d6616e19e99 Mon Sep 17 00:00:00 2001 From: Hamlin Li Date: Tue, 11 Nov 2025 14:05:43 +0000 Subject: [PATCH 12/13] Update src/hotspot/share/opto/superword.cpp refactor `is_velt_basic_type_compatible_use_def` Co-authored-by: Emanuel Peter --- src/hotspot/share/opto/superword.cpp | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/hotspot/share/opto/superword.cpp b/src/hotspot/share/opto/superword.cpp index dea8be591d7ac..c5d42444729eb 100644 --- a/src/hotspot/share/opto/superword.cpp +++ b/src/hotspot/share/opto/superword.cpp @@ -2334,8 +2334,11 @@ bool SuperWord::is_velt_basic_type_compatible_use_def(Node* use, Node* def) cons // Input size of use equals output size of def if (type2aelembytes(use_bt) == type2aelembytes(def_bt)) { return true; - } else if (VectorNode::is_different_use_def_size_supported()) { - return use->is_CMove() && def->is_Bool(); + } + + // Allow CMove to have different type for comparision and moving. + if (VectorNode::is_different_use_def_size_supported() && return use->is_CMove() && def->is_Bool()) { + return true; } return false; From a89d26c47cc24bc88e99cc71761ac2afa22fa35d Mon Sep 17 00:00:00 2001 From: hamlin Date: Tue, 11 Nov 2025 14:50:46 +0000 Subject: [PATCH 13/13] fix typo --- src/hotspot/share/opto/superword.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/hotspot/share/opto/superword.cpp b/src/hotspot/share/opto/superword.cpp index c5d42444729eb..a90d9011c7379 100644 --- a/src/hotspot/share/opto/superword.cpp +++ b/src/hotspot/share/opto/superword.cpp @@ -2337,7 +2337,7 @@ bool SuperWord::is_velt_basic_type_compatible_use_def(Node* use, Node* def) cons } // Allow CMove to have different type for comparision and moving. - if (VectorNode::is_different_use_def_size_supported() && return use->is_CMove() && def->is_Bool()) { + if (VectorNode::is_different_use_def_size_supported() && use->is_CMove() && def->is_Bool()) { return true; }