8370863: VectorAPI: Optimize the VectorMaskCast chain in specific patterns

src/hotspot/share/opto/vectornode.cpp

@@ -1046,6 +1046,25 @@ Node* VectorNode::Ideal(PhaseGVN* phase, bool can_reshape) {
   return nullptr;
 }
 
+// Traverses a chain of VectorMaskCast and returns the first non VectorMaskCast node.
+//
+// Due to the unique nature of vector masks, for specific IR patterns,
+// VectorMaskCast does not affect the output results. For example:
+//   (VectorStoreMask (VectorMaskCast ... (VectorLoadMask x))) => (x)
+//   x remains to be a bool vector with no changes.
+// This function can be used to eliminate the VectorMaskCast in such patterns.
+Node* VectorNode::uncast_mask(Node* n) {
+  while (n->Opcode() == Op_VectorMaskCast) {
+    Node* in1 = n->in(1);
+    if (!in1->isa_Vector()) {
+      break;
+    }
+    assert(n->as_Vector()->length() == in1->as_Vector()->length(), "vector length must match");
+    n = in1;
+  }
+  return n;
+}
+
 // Return initial Pack node. Additional operands added with add_opd() calls.
 PackNode* PackNode::make(Node* s, uint vlen, BasicType bt) {
   const TypeVect* vt = TypeVect::make(bt, vlen);
@@ -1452,10 +1471,11 @@ Node* VectorLoadMaskNode::Identity(PhaseGVN* phase) {
 
 Node* VectorStoreMaskNode::Identity(PhaseGVN* phase) {
   // Identity transformation on boolean vectors.
-  //   VectorStoreMask (VectorLoadMask bv) elem_size ==> bv
+  //   VectorStoreMask (VectorMaskCast ... VectorLoadMask bv) elem_size ==> bv
   //   vector[n]{bool} => vector[n]{t} => vector[n]{bool}
-  if (in(1)->Opcode() == Op_VectorLoadMask) {
-    return in(1)->in(1);
+  Node* in1 = VectorNode::uncast_mask(in(1));
+  if (in1->Opcode() == Op_VectorLoadMask && length() == in1->as_Vector()->length()) {
+    return in1->in(1);
   }
   return this;
 }
@@ -1885,11 +1905,12 @@ Node* VectorMaskOpNode::Ideal(PhaseGVN* phase, bool can_reshape) {
 }
 
 Node* VectorMaskCastNode::Identity(PhaseGVN* phase) {
-  Node* in1 = in(1);
-  // VectorMaskCast (VectorMaskCast x) => x
-  if (in1->Opcode() == Op_VectorMaskCast &&
-      vect_type()->eq(in1->in(1)->bottom_type())) {
-      return in1->in(1);
+  // (VectorMaskCast (VectorMaskCast ... x)) => (x)
+  // If the types of the input and output nodes in a VectorMaskCast chain are
+  // exactly the same, the intermediate VectorMaskCast nodes can be eliminated.
+  Node* n = VectorNode::uncast_mask(this);
+  if (vect_type()->eq(n->bottom_type())) {
+      return n;
   }
   return this;
 }

src/hotspot/share/opto/vectornode.hpp

@@ -138,6 +138,7 @@ class VectorNode : public TypeNode {
   static bool is_scalar_op_that_returns_int_but_vector_op_returns_long(int opc);
   static bool is_reinterpret_opcode(int opc);
 
+  static Node* uncast_mask(Node* n);
 
   static void trace_new_vector(Node* n, const char* context) {
 #ifdef ASSERT

test/hotspot/jtreg/compiler/vectorapi/VectorMaskCastIdentityTest.java

@@ -23,9 +23,9 @@
 
 /*
 * @test
-* @bug 8356760
+* @bug 8356760 8370863
 * @library /test/lib /
-* @summary Optimize VectorMask.fromLong for all-true/all-false cases
+* @summary test VectorMaskCast Identity() optimizations
 * @modules jdk.incubator.vector
 *
 * @run driver compiler.vectorapi.VectorMaskCastIdentityTest
@@ -49,9 +49,14 @@ public class VectorMaskCastIdentityTest {
     }
 
     @Test
-    @IR(counts = { IRNode.VECTOR_MASK_CAST, "= 2" }, applyIfCPUFeatureOr = {"asimd", "true"})
+    @IR(counts = { IRNode.VECTOR_MASK_CAST, "= 2" },
+        applyIfCPUFeature = {"sve", "true"})
+    @IR(counts = { IRNode.VECTOR_MASK_CAST, "= 0" },
+        applyIfCPUFeatureAnd = {"asimd", "true", "sve", "false"})
     public static int testTwoCastToDifferentType() {
         // The types before and after the two casts are not the same, so the cast cannot be eliminated.
+        // On NEON, VectorMaskCast nodes are eliminated by optimization pattern
+        // (VectorStoreMask (VectorMaskCast ... (VectorLoadMask x))) => x
         VectorMask<Float> mFloat64 = VectorMask.fromArray(FloatVector.SPECIES_64, mr, 0);
         VectorMask<Double> mDouble128 = mFloat64.cast(DoubleVector.SPECIES_128);
         VectorMask<Integer> mInt64 = mDouble128.cast(IntVector.SPECIES_64);
@@ -66,7 +71,10 @@ public static void testTwoCastToDifferentType_runner() {
     }
 
     @Test
-    @IR(counts = { IRNode.VECTOR_MASK_CAST, "= 2" }, applyIfCPUFeatureOr = {"avx2", "true"})
+    @IR(counts = { IRNode.VECTOR_MASK_CAST, "= 2" },
+        applyIfCPUFeature = {"avx512", "true"})
+    @IR(counts = { IRNode.VECTOR_MASK_CAST, "= 0" },
+        applyIfCPUFeatureAnd = {"avx2", "true", "avx512", "false"})
     public static int testTwoCastToDifferentType2() {
         // The types before and after the two casts are not the same, so the cast cannot be eliminated.
         VectorMask<Integer> mInt128 = VectorMask.fromArray(IntVector.SPECIES_128, mr, 0);
@@ -83,10 +91,11 @@ public static void testTwoCastToDifferentType2_runner() {
     }
 
     @Test
-    @IR(counts = { IRNode.VECTOR_MASK_CAST, "= 0" }, applyIfCPUFeatureOr = {"avx2", "true", "asimd", "true"})
+    @IR(counts = { IRNode.VECTOR_MASK_CAST, "= 0" },
+        applyIfCPUFeatureOr = {"avx2", "true", "asimd", "true"})
     public static int testTwoCastToSameType() {
         // The types before and after the two casts are the same, so the cast will be eliminated.
-        VectorMask<Integer> mInt128 = VectorMask.fromArray(IntVector.SPECIES_128, mr, 0);
+        VectorMask<Integer> mInt128 = VectorMask.fromArray(IntVector.SPECIES_128, mr, 0).not();
         VectorMask<Float> mFloat128 = mInt128.cast(FloatVector.SPECIES_128);
         VectorMask<Integer> mInt128_2 = mFloat128.cast(IntVector.SPECIES_128);
         return mInt128_2.trueCount();
@@ -95,23 +104,41 @@ public static int testTwoCastToSameType() {
     @Run(test = "testTwoCastToSameType")
     public static void testTwoCastToSameType_runner() {
         int count = testTwoCastToSameType();
-        VectorMask<Integer> mInt128 = VectorMask.fromArray(IntVector.SPECIES_128, mr, 0);
+        VectorMask<Integer> mInt128 = VectorMask.fromArray(IntVector.SPECIES_128, mr, 0).not();
         Asserts.assertEquals(count, mInt128.trueCount());
     }
 
     @Test
-    @IR(counts = { IRNode.VECTOR_MASK_CAST, "= 1" }, applyIfCPUFeatureOr = {"avx2", "true", "asimd", "true"})
+    @IR(counts = { IRNode.VECTOR_MASK_CAST, "= 1" },
+        applyIfCPUFeatureOr = {"avx2", "true", "asimd", "true"})
     public static int testOneCastToDifferentType() {
         // The types before and after the only cast are different, the cast will not be eliminated.
-        VectorMask<Float> mFloat128 = VectorMask.fromArray(FloatVector.SPECIES_128, mr, 0).not();
-        VectorMask<Integer> mInt128 = mFloat128.cast(IntVector.SPECIES_128);
-        return mInt128.trueCount();
+        VectorMask<Integer> mInt128 = VectorMask.fromArray(IntVector.SPECIES_128, mr, 0).not();
+        VectorMask<Short> mShort64 = mInt128.cast(ShortVector.SPECIES_64);
+        return mShort64.trueCount();
     }
 
     @Run(test = "testOneCastToDifferentType")
     public static void testOneCastToDifferentType_runner() {
         int count = testOneCastToDifferentType();
-        VectorMask<Float> mInt128 = VectorMask.fromArray(FloatVector.SPECIES_128, mr, 0).not();
+        VectorMask<Integer> mInt128 = VectorMask.fromArray(IntVector.SPECIES_128, mr, 0).not();
+        Asserts.assertEquals(count, mInt128.trueCount());
+    }
+
+    @Test
+    @IR(counts = { IRNode.VECTOR_MASK_CAST, "= 0" },
+        applyIfCPUFeatureOr = {"avx2", "true", "asimd", "true"})
+    public static int testOneCastToSameType() {
+        // The types before and after the only cast are the same, the cast will be eliminated.
+        VectorMask<Integer> mInt128 = VectorMask.fromArray(IntVector.SPECIES_128, mr, 0).not();
+        mInt128 = mInt128.cast(IntVector.SPECIES_128);
+        return mInt128.trueCount();
+    }
+
+    @Run(test = "testOneCastToSameType")
+    public static void testOneCastToSameType_runner() {
+        int count = testOneCastToSameType();
+        VectorMask<Integer> mInt128 = VectorMask.fromArray(IntVector.SPECIES_128, mr, 0).not();
         Asserts.assertEquals(count, mInt128.trueCount());
     }