Intrinsify Reference.clear (JDK-8329597)

compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/core/common/memory/BarrierType.java

@@ -60,6 +60,13 @@ public enum BarrierType {
      */
     POST_INIT_WRITE,
 
+    /**
+     * Clear {@link java.lang.ref.Reference}.referent. In the HotSpot world this corresponds to a
+     * write decorated with {@code AS_NO_KEEPALIVE}. Depending on the particular garbage collector
+     * this might do something different than {@link #FIELD}.
+     */
+    AS_NO_KEEPALIVE_WRITE,
+
     /**
      * Read of {@link java.lang.ref.Reference}.referent. In the HotSpot world this corresponds to an
      * access decorated with {@code ON_WEAK_OOP_REF}. Depending on the particular garbage collector

compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/GraalHotSpotVMConfig.java

@@ -83,6 +83,10 @@ public boolean useG1GC() {
         return gc == HotSpotGraalRuntime.HotSpotGC.G1;
     }
 
+    public boolean useXGC() {
+        return gc == HotSpotGraalRuntime.HotSpotGC.X;
+    }
+
     public final HotSpotGraalRuntime.HotSpotGC gc = getSelectedGC();
 
     private HotSpotGraalRuntime.HotSpotGC getSelectedGC() throws GraalError {
@@ -585,6 +589,8 @@ private long getZGCAddressField(String name) {
                     "ZBarrierSetRuntime::no_keepalive_load_barrier_on_weak_oop_field_preloaded");
     public final long zBarrierSetRuntimeNoKeepaliveLoadBarrierOnPhantomOopFieldPreloaded = getZGCAddressField(
                     "ZBarrierSetRuntime::no_keepalive_load_barrier_on_phantom_oop_field_preloaded");
+    public final long zBarrierSetRuntimeNoKeepaliveStoreBarrierOnOopFieldWithoutHealing = getZGCAddressField(
+                    "ZBarrierSetRuntime::no_keepalive_store_barrier_on_oop_field_without_healing");
     public final long zBarrierSetRuntimeStoreBarrierOnNativeOopFieldWithoutHealing = getZGCAddressField("ZBarrierSetRuntime::store_barrier_on_native_oop_field_without_healing");
     public final long zBarrierSetRuntimeStoreBarrierOnOopFieldWithHealing = getZGCAddressField("ZBarrierSetRuntime::store_barrier_on_oop_field_with_healing");
     public final long zBarrierSetRuntimeStoreBarrierOnOopFieldWithoutHealing = getZGCAddressField("ZBarrierSetRuntime::store_barrier_on_oop_field_without_healing");

compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/ZWriteBarrierSetLIRGeneratorTool.java

@@ -111,6 +111,9 @@ default ForeignCallLinkage getWriteBarrierStub(BarrierType barrierType, StoreKin
                     callTarget = getForeignCalls().lookupForeignCall(HotSpotHostForeignCallsProvider.Z_STORE_BARRIER_WITHOUT_HEALING);
                 }
                 break;
+            case AS_NO_KEEPALIVE_WRITE:
+                callTarget = getForeignCalls().lookupForeignCall(HotSpotHostForeignCallsProvider.Z_REFERENCE_CLEAR_BARRIER);
+                break;
             default:
                 throw GraalError.shouldNotReachHere("Unexpected barrier type: " + barrierType);
         }

compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/meta/HotSpotGraphBuilderPlugins.java

@@ -265,7 +265,7 @@ public void run() {
                 registerArrayPlugins(invocationPlugins, replacements, config);
                 registerStringPlugins(invocationPlugins, replacements, wordTypes, foreignCalls, config);
                 registerArraysSupportPlugins(invocationPlugins, replacements, target.arch);
-                registerReferencePlugins(invocationPlugins, replacements);
+                registerReferencePlugins(invocationPlugins, config, replacements);
                 registerTrufflePlugins(invocationPlugins, wordTypes, config);
                 registerInstrumentationImplPlugins(invocationPlugins, config, replacements);
                 for (HotSpotInvocationPluginProvider p : GraalServices.load(HotSpotInvocationPluginProvider.class)) {
@@ -1366,7 +1366,7 @@ public boolean isGraalOnly() {
         });
     }
 
-    private static void registerReferencePlugins(InvocationPlugins plugins, Replacements replacements) {
+    private static void registerReferencePlugins(InvocationPlugins plugins, GraalHotSpotVMConfig config, Replacements replacements) {
         Registration r = new Registration(plugins, Reference.class, replacements);
         r.register(new ReachabilityFencePlugin() {
             @Override
@@ -1386,6 +1386,23 @@ public boolean apply(GraphBuilderContext b, ResolvedJavaMethod targetMethod, Rec
                 return true;
             }
         });
+        if (JavaVersionUtil.JAVA_SPEC >= 24 && !config.useXGC()) {
+            r.register(new InlineOnlyInvocationPlugin("clear0", Receiver.class) {
+                @Override
+                public boolean apply(GraphBuilderContext b, ResolvedJavaMethod targetMethod, Receiver receiver) {
+                    try (InvocationPluginHelper helper = new InvocationPluginHelper(b, targetMethod)) {
+                        ValueNode offset = b.add(ConstantNode.forLong(HotSpotReplacementsUtil.referentOffset(b.getMetaAccess())));
+                        AddressNode address = b.add(new OffsetAddressNode(receiver.get(true), offset));
+                        FieldLocationIdentity locationIdentity = new FieldLocationIdentity(HotSpotReplacementsUtil.referentField(b.getMetaAccess()));
+                        JavaReadNode referent = b.add(new JavaReadNode(StampFactory.object(), JavaKind.Object, address, locationIdentity, BarrierType.WEAK_REFERS_TO, MemoryOrderMode.PLAIN, true));
+                        helper.emitReturnIf(IsNullNode.create(referent), null, GraalDirectives.LIKELY_PROBABILITY);
+                        b.add(new JavaWriteNode(JavaKind.Object, address, locationIdentity, ConstantNode.defaultForKind(JavaKind.Object), BarrierType.AS_NO_KEEPALIVE_WRITE, true));
+                        helper.emitFinalReturn(JavaKind.Void, null);
+                        return true;
+                    }
+                }
+            });
+        }
         r = new Registration(plugins, PhantomReference.class, replacements);
         r.register(new InlineOnlyInvocationPlugin("refersTo0", Receiver.class, Object.class) {
             @Override
@@ -1399,6 +1416,23 @@ public boolean apply(GraphBuilderContext b, ResolvedJavaMethod targetMethod, Rec
                 return true;
             }
         });
+        if (JavaVersionUtil.JAVA_SPEC >= 24 && !config.useXGC()) {
+            r.register(new InlineOnlyInvocationPlugin("clear0", Receiver.class) {
+                @Override
+                public boolean apply(GraphBuilderContext b, ResolvedJavaMethod targetMethod, Receiver receiver) {
+                    try (InvocationPluginHelper helper = new InvocationPluginHelper(b, targetMethod)) {
+                        ValueNode offset = b.add(ConstantNode.forLong(HotSpotReplacementsUtil.referentOffset(b.getMetaAccess())));
+                        AddressNode address = b.add(new OffsetAddressNode(receiver.get(true), offset));
+                        FieldLocationIdentity locationIdentity = new FieldLocationIdentity(HotSpotReplacementsUtil.referentField(b.getMetaAccess()));
+                        JavaReadNode referent = b.add(new JavaReadNode(StampFactory.object(), JavaKind.Object, address, locationIdentity, BarrierType.PHANTOM_REFERS_TO, MemoryOrderMode.PLAIN, true));
+                        helper.emitReturnIf(IsNullNode.create(referent), null, GraalDirectives.LIKELY_PROBABILITY);
+                        b.add(new JavaWriteNode(JavaKind.Object, address, locationIdentity, ConstantNode.defaultForKind(JavaKind.Object), BarrierType.AS_NO_KEEPALIVE_WRITE, true));
+                        helper.emitFinalReturn(JavaKind.Void, null);
+                        return true;
+                    }
+                }
+            });
+        }
     }
 
     private static void registerInstrumentationImplPlugins(InvocationPlugins plugins, GraalHotSpotVMConfig config, Replacements replacements) {

compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/meta/HotSpotHostForeignCallsProvider.java

@@ -226,6 +226,10 @@ public abstract class HotSpotHostForeignCallsProvider extends HotSpotForeignCall
     public static final HotSpotForeignCallDescriptor Z_REFERENCE_GET_BARRIER = new HotSpotForeignCallDescriptor(LEAF_NO_VZERO, HAS_SIDE_EFFECT, NO_LOCATIONS,
                     "ZBarrierSetRuntime::load_barrier_on_weak_oop_field_preloaded", long.class, long.class, long.class);
 
+    // void no_keepalive_store_barrier_on_oop_field_without_healing(oop* p);
+    public static final HotSpotForeignCallDescriptor Z_REFERENCE_CLEAR_BARRIER = new HotSpotForeignCallDescriptor(LEAF_NO_VZERO, HAS_SIDE_EFFECT, NO_LOCATIONS,
+                    "ZBarrierSetRuntime::no_keepalive_store_barrier_on_oop_field_without_healing", void.class, long.class);
+
     // oopDesc* no_keepalive_load_barrier_on_weak_oop_field_preloaded(oopDesc* o, oop* p);
     public static final HotSpotForeignCallDescriptor Z_WEAK_REFERS_TO_BARRIER = new HotSpotForeignCallDescriptor(LEAF_NO_VZERO, HAS_SIDE_EFFECT, NO_LOCATIONS,
                     "ZBarrierSetRuntime::no_keepalive_load_barrier_on_weak_oop_field_preloaded", long.class, long.class, long.class);
@@ -579,6 +583,7 @@ public void initialize(HotSpotProviders providers, OptionValues options) {
         linkStackOnlyForeignCall(c.gc == Z, options, providers, Z_STORE_BARRIER_WITH_HEALING, c.zBarrierSetRuntimeStoreBarrierOnOopFieldWithHealing, DONT_PREPEND_THREAD);
         linkStackOnlyForeignCall(c.gc == Z, options, providers, Z_STORE_BARRIER_NATIVE, c.zBarrierSetRuntimeStoreBarrierOnNativeOopFieldWithoutHealing, DONT_PREPEND_THREAD);
         linkStackOnlyForeignCall(c.gc == Z, options, providers, Z_REFERENCE_GET_BARRIER, c.zBarrierSetRuntimeLoadBarrierOnWeakOopFieldPreloaded, DONT_PREPEND_THREAD);
+        linkStackOnlyForeignCall(c.gc == Z, options, providers, Z_REFERENCE_CLEAR_BARRIER, c.zBarrierSetRuntimeNoKeepaliveStoreBarrierOnOopFieldWithoutHealing, DONT_PREPEND_THREAD);
         linkStackOnlyForeignCall(c.gc == Z, options, providers, Z_WEAK_REFERS_TO_BARRIER, c.zBarrierSetRuntimeNoKeepaliveLoadBarrierOnWeakOopFieldPreloaded, DONT_PREPEND_THREAD);
         linkStackOnlyForeignCall(c.gc == Z, options, providers, Z_PHANTOM_REFERS_TO_BARRIER, c.zBarrierSetRuntimeNoKeepaliveLoadBarrierOnPhantomOopFieldPreloaded, DONT_PREPEND_THREAD);
         linkStackOnlyForeignCall(c.gc == Z, options, providers, Z_ARRAY_BARRIER, c.zBarrierSetRuntimeLoadBarrierOnOopArray, DONT_PREPEND_THREAD);

compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/nodes/gc/CardTableBarrierSet.java

@@ -204,6 +204,7 @@ public boolean needsWriteBarrier(FixedAccessNode node, ValueNode writtenValue) {
             case NONE:
                 return false;
             case FIELD:
+            case AS_NO_KEEPALIVE_WRITE:
             case ARRAY:
             case UNKNOWN:
                 return writeRequiresBarrier(node, writtenValue);
@@ -234,7 +235,7 @@ private static boolean hasWriteBarrier(ArrayRangeWrite write) {
     private static void addSerialPostWriteBarrier(FixedAccessNode node, AddressNode address, StructuredGraph graph) {
         // Use a precise barrier for everything that might be an array write. Being too precise with
         // the barriers does not cause any correctness issues.
-        boolean precise = node.getBarrierType() != BarrierType.FIELD;
+        boolean precise = node.getBarrierType() != BarrierType.FIELD && node.getBarrierType() != BarrierType.AS_NO_KEEPALIVE_WRITE;
         graph.addAfterFixed(node, graph.add(new SerialWriteBarrierNode(address, precise)));
     }
 

compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/nodes/gc/G1BarrierSet.java

@@ -168,12 +168,14 @@ private void addWriteBarriers(FixedAccessNode node, ValueNode writtenValue, Valu
             case FIELD:
             case ARRAY:
             case UNKNOWN:
+            case AS_NO_KEEPALIVE_WRITE:
                 if (isObjectValue(writtenValue)) {
                     StructuredGraph graph = node.graph();
                     boolean init = node.getLocationIdentity().isInit();
-                    if (!init) {
+                    if (!init && barrierType != BarrierType.AS_NO_KEEPALIVE_WRITE) {
                         // The pre barrier does nothing if the value being read is null, so it can
                         // be explicitly skipped when this is an initializing store.
+                        // No keep-alive means no need for the pre-barrier.
                         addG1PreWriteBarrier(node, node.getAddress(), expectedValue, doLoad, graph);
                     }
                     if (writeRequiresPostBarrier(node, writtenValue)) {

compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/nodes/memory/WriteNode.java

@@ -72,7 +72,7 @@ protected WriteNode(NodeClass<? extends WriteNode> c, AddressNode address, Locat
                     MemoryOrderMode memoryOrder) {
         super(c, address, location, value, barrierType);
         assert barrierType == BarrierType.NONE || barrierType == BarrierType.ARRAY || barrierType == BarrierType.FIELD || barrierType == BarrierType.UNKNOWN ||
-                        barrierType == BarrierType.POST_INIT_WRITE : barrierType;
+                        barrierType == BarrierType.POST_INIT_WRITE || barrierType == BarrierType.AS_NO_KEEPALIVE_WRITE : barrierType;
         this.killedLocationIdentity = killedLocationIdentity;
         this.memoryOrder = memoryOrder;
     }

substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/Target_java_lang_ref_PhantomReference.java

@@ -29,11 +29,20 @@
 import com.oracle.svm.core.SubstrateUtil;
 import com.oracle.svm.core.annotate.Substitute;
 import com.oracle.svm.core.annotate.TargetClass;
+import com.oracle.svm.core.annotate.TargetElement;
+import com.oracle.svm.core.jdk.JDKLatest;
 
 @TargetClass(PhantomReference.class)
 public final class Target_java_lang_ref_PhantomReference<T> {
     @Substitute
     boolean refersTo0(Object obj) {
         return ReferenceInternals.refersTo(SubstrateUtil.cast(this, PhantomReference.class), obj);
     }
+
+    @Substitute
+    @TargetElement(onlyWith = JDKLatest.class)
+    private void clear0() {
+        ReferenceInternals.clear(SubstrateUtil.cast(this, PhantomReference.class));
+    }
+
 }

substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/Target_java_lang_ref_Reference.java

@@ -43,6 +43,8 @@
 import com.oracle.svm.core.annotate.RecomputeFieldValue;
 import com.oracle.svm.core.annotate.Substitute;
 import com.oracle.svm.core.annotate.TargetClass;
+import com.oracle.svm.core.annotate.TargetElement;
+import com.oracle.svm.core.jdk.JDKLatest;
 import com.oracle.svm.core.util.VMError;
 import com.oracle.svm.util.ReflectionUtil;
 
@@ -123,16 +125,18 @@ public final class Target_java_lang_ref_Reference<T> {
     @KeepOriginal
     native T get();
 
-    @Substitute
-    public void clear() {
-        ReferenceInternals.clear(SubstrateUtil.cast(this, Reference.class));
-    }
+    @KeepOriginal
+    native void clear();
 
     @Substitute
     private void clear0() {
-        clear();
+        ReferenceInternals.clear(SubstrateUtil.cast(this, Reference.class));
     }
 
+    @TargetElement(onlyWith = JDKLatest.class)
+    @KeepOriginal
+    native void clearImpl();
+
     @KeepOriginal
     native boolean refersToImpl(T obj);