diff --git a/common.json b/common.json index 099f0f0beaf9..b4e32e6ed026 100644 --- a/common.json +++ b/common.json @@ -8,7 +8,7 @@ "COMMENT.jdks": "When adding or removing JDKs keep in sync with JDKs in ci/common.jsonnet", "jdks": { - "galahad-jdk": {"name": "jpg-jdk", "version": "24", "build_id": "jdk-24+19-2105", "platformspecific": true, "extrabundles": ["static-libs"]}, + "galahad-jdk": {"name": "jpg-jdk", "version": "24", "build_id": "2024-09-27-1322328.yudi.zheng.jdk", "platformspecific": true, "extrabundles": ["static-libs"]}, "oraclejdk17": {"name": "jpg-jdk", "version": "17.0.7", "build_id": "jdk-17.0.7+8", "platformspecific": true, "extrabundles": ["static-libs"]}, "labsjdk-ce-17": {"name": "labsjdk", "version": "ce-17.0.7+4-jvmci-23.1-b02", "platformspecific": true }, diff --git a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/GraalHotSpotVMConfig.java b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/GraalHotSpotVMConfig.java index 4f63e1596064..f0db3c85ebf0 100644 --- a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/GraalHotSpotVMConfig.java +++ b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/GraalHotSpotVMConfig.java @@ -56,7 +56,6 @@ public class GraalHotSpotVMConfig extends GraalHotSpotVMConfigAccess { GraalHotSpotVMConfig(HotSpotVMConfigStore store) { super(store); - assert narrowKlassShift <= logKlassAlignment : Assertions.errorMessageContext("narrowKlassShift", narrowKlassShift, "logKlassAlignment", logKlassAlignment); int logMinObjAlignment = logMinObjAlignment(); assert narrowOopShift <= logMinObjAlignment : Assertions.errorMessageContext("narrowOopShift", narrowOopShift, "logMinObjAlignment", logMinObjAlignment); oopEncoding = new CompressEncoding(narrowOopBase, narrowOopShift); @@ -187,6 +186,11 @@ public long gcTotalCollectionsAddress() { // Compressed Oops related values. public final boolean useCompressedOops = getFlag("UseCompressedOops", Boolean.class); public final boolean useCompressedClassPointers = getFlag("UseCompressedClassPointers", Boolean.class); + // JDK-8305895 allows storing the compressed class pointer in the upper 22 bits of the mark + // word. This runtime optimization is guarded by the flag UseCompactObjectHeaders. It depends + // on compressed class pointers, meaning that if useCompactObjectHeaders is true, + // useCompressedClassPointers is certainly true. + public final boolean useCompactObjectHeaders = getFlag("UseCompactObjectHeaders", Boolean.class, false, JDK >= 24); public final long narrowOopBase = getFieldValue("CompilerToVM::Data::Universe_narrow_oop_base", Long.class, "address"); public final int narrowOopShift = getFieldValue("CompilerToVM::Data::Universe_narrow_oop_shift", Integer.class, "int"); @@ -199,7 +203,6 @@ public final int logMinObjAlignment() { public final int narrowKlassSize = getFieldValue("CompilerToVM::Data::sizeof_narrowKlass", Integer.class, "int"); public final long narrowKlassBase = getFieldValue("CompilerToVM::Data::Universe_narrow_klass_base", Long.class, "address"); public final int narrowKlassShift = getFieldValue("CompilerToVM::Data::Universe_narrow_klass_shift", Integer.class, "int"); - public final int logKlassAlignment = getConstant("LogKlassAlignmentInBytes", Integer.class); public final int stackShadowPages = getFlag("StackShadowPages", Integer.class); public final int vmPageSize = getFieldValue("CompilerToVM::Data::vm_page_size", Integer.class, "size_t"); @@ -207,7 +210,8 @@ public final int logMinObjAlignment() { public final int softwarePrefetchHintDistance = getFlag("SoftwarePrefetchHintDistance", Integer.class, -1, "aarch64".equals(osArch)); public final int markOffset = getFieldOffset("oopDesc::_mark", Integer.class, markWord); - public final int hubOffset = getFieldOffset("oopDesc::_metadata._klass", Integer.class, "Klass*"); + // TODO remove before merge + public final int hubOffset = useCompactObjectHeaders ? 0xBAADCAFE : getFieldOffset("oopDesc::_metadata._klass", Integer.class, "Klass*"); public final int superCheckOffsetOffset = getFieldOffset("Klass::_super_check_offset", Integer.class, "juint"); public final int secondarySuperCacheOffset = getFieldOffset("Klass::_secondary_super_cache", Integer.class, "Klass*"); @@ -238,12 +242,17 @@ public final int logMinObjAlignment() { public final int arrayOopDescSize = getFieldValue("CompilerToVM::Data::sizeof_arrayOopDesc", Integer.class, "int"); + public final int arrayLengthOffsetInBytes = getFieldValue("CompilerToVM::Data::arrayOopDesc_length_offset_in_bytes", Integer.class, "int", -1, JDK >= 24); + /** * The offset of the array length word in an array object's header. * * See {@code arrayOopDesc::length_offset_in_bytes()}. */ public final int arrayOopDescLengthOffset() { + if (JDK >= 24) { + return arrayLengthOffsetInBytes; + } return useCompressedClassPointers ? hubOffset + narrowKlassSize : arrayOopDescSize; } @@ -348,9 +357,22 @@ public int threadLastJavaFpOffset() { public final int frameInterpreterFrameSenderSpOffset = getConstant("frame::interpreter_frame_sender_sp_offset", Integer.class, 0, osArch.equals("amd64")); public final int frameInterpreterFrameLastSpOffset = getConstant("frame::interpreter_frame_last_sp_offset", Integer.class, 0, osArch.equals("amd64")); - public final int lockMaskInPlace = getConstant("markWord::lock_mask_in_place", Integer.class); + public final long markWordLockMaskInPlace = getConstant("markWord::lock_mask_in_place", Long.class); + public final long markWordHashMask = getConstant("markWord::hash_mask", Long.class); + + public final long markWordNoHashInPlace = getConstant("markWord::no_hash_in_place", Long.class); + public final long markWordNoLockInPlace = getConstant("markWord::no_lock_in_place", Long.class); + + // Mark word right shift to get identity hash code. + public final int markWordHashCodeShift = getConstant("markWord::hash_shift", Integer.class); + // Mark word right shift to get compressed klass pointer + public final int markWordKlassShift = getConstant("markWord::klass_shift", Integer.class, 0, JDK >= 24); + + // The following three constants are declared as 64 bits uintptr_t, but known to be 32 bits public final int unlockedValue = getConstant("markWord::unlocked_value", Integer.class); public final int monitorValue = getConstant("markWord::monitor_value", Integer.class); + // Identity hash code value when uninitialized. + public final int uninitializedIdentityHashCodeValue = getConstant("markWord::no_hash", Integer.class); // This field has no type in vmStructs.cpp public final int objectMonitorOwner = getFieldOffset("ObjectMonitor::_owner", Integer.class, null); @@ -359,26 +381,9 @@ public int threadLastJavaFpOffset() { public final int objectMonitorEntryList = getFieldOffset("ObjectMonitor::_EntryList", Integer.class, "ObjectWaiter*"); public final int objectMonitorSucc = getFieldOffset("ObjectMonitor::_succ", Integer.class, "JavaThread*"); - public final int markWordNoHashInPlace = getConstant("markWord::no_hash_in_place", Integer.class); - public final int markWordNoLockInPlace = getConstant("markWord::no_lock_in_place", Integer.class); - - public long defaultPrototypeMarkWord() { - return this.markWordNoHashInPlace | this.markWordNoLockInPlace; - } - - /** - * Mark word right shift to get identity hash code. - */ - public final int identityHashCodeShift = getConstant("markWord::hash_shift", Integer.class); - public final int contEntry = getFieldOffset("JavaThread::_cont_entry", Integer.class, "ContinuationEntry*", -1, JDK >= 24); public final int pinCount = getFieldOffset("ContinuationEntry::_pin_count", Integer.class, "uint32_t", -1, JDK >= 24); - /** - * Identity hash code value when uninitialized. - */ - public final int uninitializedIdentityHashCodeValue = getConstant("markWord::no_hash", Integer.class); - public final int methodCompiledEntryOffset = getFieldOffset("Method::_from_compiled_entry", Integer.class, "address"); public final int compilationLevelFullOptimization = getConstant("CompLevel_full_optimization", Integer.class); diff --git a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/aarch64/AArch64HotSpotBackend.java b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/aarch64/AArch64HotSpotBackend.java index dd278bd30654..1c0013ba3923 100644 --- a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/aarch64/AArch64HotSpotBackend.java +++ b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/aarch64/AArch64HotSpotBackend.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2013, 2023, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2013, 2024, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -419,15 +419,18 @@ private void emitCodePrefix(CompilationResultBuilder crb, ResolvedJavaMethod ins CallingConvention cc = regConfig.getCallingConvention(HotSpotCallingConventionType.JavaCallee, null, parameterTypes, this); Register receiver = asRegister(cc.getArgument(0)); int size = config.useCompressedClassPointers ? 32 : 64; - AArch64Address klassAddress = masm.makeAddress(size, receiver, config.hubOffset); if (config.icSpeculatedKlassOffset == Integer.MAX_VALUE) { crb.recordMark(HotSpotMarkId.UNVERIFIED_ENTRY); Register klass = rscratch1; if (config.useCompressedClassPointers) { - masm.ldr(32, klass, klassAddress); + if (config.useCompactObjectHeaders) { + ((AArch64HotSpotMacroAssembler) masm).loadCompactClassPointer(klass, receiver); + } else { + masm.ldr(size, klass, masm.makeAddress(size, receiver, config.hubOffset)); + } AArch64HotSpotMove.decodeKlassPointer(masm, klass, klass, config.getKlassEncoding()); } else { - masm.ldr(64, klass, klassAddress); + masm.ldr(size, klass, masm.makeAddress(size, receiver, config.hubOffset)); } // c1_LIRAssembler_aarch64.cpp: const Register IC_Klass = rscratch2; Register inlineCacheKlass = AArch64HotSpotRegisterConfig.inlineCacheRegister; @@ -436,7 +439,6 @@ private void emitCodePrefix(CompilationResultBuilder crb, ResolvedJavaMethod ins masm.branchConditionally(AArch64Assembler.ConditionFlag.EQ, verifiedStub); AArch64Call.directJmp(crb, masm, getForeignCalls().lookupForeignCall(IC_MISS_HANDLER)); } else { - // JDK-8322630 (removed ICStubs) Register data = AArch64HotSpotRegisterConfig.inlineCacheRegister; Register tmp1 = rscratch1; @@ -445,16 +447,24 @@ private void emitCodePrefix(CompilationResultBuilder crb, ResolvedJavaMethod ins // Size of IC check sequence checked with a guarantee below. int inlineCacheCheckSize = AArch64Call.isNearCall(icMissHandler) ? 20 : 32; + if (config.useCompactObjectHeaders) { + // Extra instruction for shifting + inlineCacheCheckSize += 4; + } masm.align(config.codeEntryAlignment, masm.position() + inlineCacheCheckSize); int startICCheck = masm.position(); crb.recordMark(HotSpotMarkId.UNVERIFIED_ENTRY); AArch64Address icSpeculatedKlass = masm.makeAddress(size, data, config.icSpeculatedKlassOffset); - masm.ldr(size, tmp1, klassAddress); + if (config.useCompactObjectHeaders) { + ((AArch64HotSpotMacroAssembler) masm).loadCompactClassPointer(tmp1, receiver); + } else { + masm.ldr(size, tmp1, masm.makeAddress(size, receiver, config.hubOffset)); + } + masm.ldr(size, tmp2, icSpeculatedKlass); masm.cmp(size, tmp1, tmp2); - masm.branchConditionally(AArch64Assembler.ConditionFlag.EQ, verifiedStub); AArch64Call.directJmp(crb, masm, icMissHandler); diff --git a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/aarch64/AArch64HotSpotMacroAssembler.java b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/aarch64/AArch64HotSpotMacroAssembler.java index 21456991370b..982013fa6d78 100644 --- a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/aarch64/AArch64HotSpotMacroAssembler.java +++ b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/aarch64/AArch64HotSpotMacroAssembler.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2022, 2024, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -29,6 +29,7 @@ import jdk.graal.compiler.asm.aarch64.AArch64Assembler; import jdk.graal.compiler.asm.aarch64.AArch64MacroAssembler; import jdk.graal.compiler.core.common.CompressEncoding; +import jdk.graal.compiler.debug.GraalError; import jdk.graal.compiler.hotspot.GraalHotSpotVMConfig; import jdk.graal.compiler.lir.aarch64.AArch64Move; import jdk.vm.ci.aarch64.AArch64; @@ -81,23 +82,24 @@ public void verifyOop(Register value, Register tmp, Register tmp2, boolean compr cbz(compressed ? 32 : 64, value, ok); } - AArch64Address hubAddress; - int hubSize = config.useCompressedClassPointers ? 32 : 64; + Register object = value; if (compressed) { CompressEncoding encoding = config.getOopEncoding(); mov(32, tmp, value); AArch64Move.UncompressPointerOp.emitUncompressCode(this, tmp, tmp, encoding, true, heapBaseRegister, false); - hubAddress = makeAddress(hubSize, tmp, config.hubOffset); - } else { - hubAddress = makeAddress(hubSize, value, config.hubOffset); + object = tmp; } // Load the class if (config.useCompressedClassPointers) { - ldr(32, tmp, hubAddress); + if (config.useCompactObjectHeaders) { + loadCompactClassPointer(tmp, object); + } else { + ldr(32, tmp, makeAddress(32, object, config.hubOffset)); + } AArch64HotSpotMove.decodeKlassPointer(this, tmp, tmp, config.getKlassEncoding()); } else { - ldr(64, tmp, hubAddress); + ldr(64, tmp, makeAddress(64, object, config.hubOffset)); } // Klass::_super_check_offset ldr(32, tmp2, makeAddress(32, tmp, config.superCheckOffsetOffset)); @@ -127,4 +129,9 @@ public void verifyHeapBase() { } } + public void loadCompactClassPointer(Register result, Register receiver) { + GraalError.guarantee(config.useCompactObjectHeaders, "Load class pointer from markWord only when UseCompactObjectHeaders is on"); + ldr(64, result, makeAddress(64, receiver, config.markOffset)); + lsr(64, result, result, config.markWordKlassShift); + } } diff --git a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/amd64/AMD64HotSpotBackend.java b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/amd64/AMD64HotSpotBackend.java index 594205adfcd5..a25cac73fb79 100644 --- a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/amd64/AMD64HotSpotBackend.java +++ b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/amd64/AMD64HotSpotBackend.java @@ -337,7 +337,6 @@ public void emitCodePrefix(ResolvedJavaMethod installedCodeOwner, CompilationRes JavaType[] parameterTypes = {providers.getMetaAccess().lookupJavaType(Object.class)}; CallingConvention cc = regConfig.getCallingConvention(HotSpotCallingConventionType.JavaCallee, null, parameterTypes, this); Register receiver = asRegister(cc.getArgument(0)); - AMD64Address src = new AMD64Address(receiver, config.hubOffset); int before; if (config.icSpeculatedKlassOffset == Integer.MAX_VALUE) { crb.recordMark(HotSpotMarkId.UNVERIFIED_ENTRY); @@ -347,18 +346,23 @@ public void emitCodePrefix(ResolvedJavaMethod installedCodeOwner, CompilationRes if (config.useCompressedClassPointers) { Register register = r10; Register heapBase = providers.getRegisters().getHeapBaseRegister(); - AMD64HotSpotMove.decodeKlassPointer(asm, register, heapBase, src, config); + if (config.useCompactObjectHeaders) { + ((AMD64HotSpotMacroAssembler) asm).loadCompactClassPointer(register, receiver); + } else { + asm.movl(register, new AMD64Address(receiver, config.hubOffset)); + } + AMD64HotSpotMove.decodeKlassPointer(asm, register, heapBase, config); if (config.narrowKlassBase != 0) { // The heap base register was destroyed above, so restore it if (config.narrowOopBase == 0L) { - asm.xorq(heapBase, heapBase); + asm.xorl(heapBase, heapBase); } else { asm.movq(heapBase, config.narrowOopBase); } } before = asm.cmpqAndJcc(inlineCacheKlass, register, ConditionFlag.NotEqual, null, false); } else { - before = asm.cmpqAndJcc(inlineCacheKlass, src, ConditionFlag.NotEqual, null, false); + before = asm.cmpqAndJcc(inlineCacheKlass, new AMD64Address(receiver, config.hubOffset), ConditionFlag.NotEqual, null, false); } crb.recordDirectCall(before, asm.position(), getForeignCalls().lookupForeignCall(IC_MISS_HANDLER), null); } else { @@ -376,6 +380,10 @@ public void emitCodePrefix(ResolvedJavaMethod installedCodeOwner, CompilationRes */ inlineCacheCheckSize += 3 + 3; } + if (config.useCompactObjectHeaders) { + // 4 bytes for extra shift instruction, 1 byte less for 0-displacement address + inlineCacheCheckSize += 3; + } asm.align(config.codeEntryAlignment, asm.position() + inlineCacheCheckSize); int startICCheck = asm.position(); @@ -384,10 +392,14 @@ public void emitCodePrefix(ResolvedJavaMethod installedCodeOwner, CompilationRes AMD64BaseAssembler.OperandSize size; if (config.useCompressedClassPointers) { - asm.movl(temp, src); + if (config.useCompactObjectHeaders) { + ((AMD64HotSpotMacroAssembler) asm).loadCompactClassPointer(temp, receiver); + } else { + asm.movl(temp, new AMD64Address(receiver, config.hubOffset)); + } size = AMD64BaseAssembler.OperandSize.DWORD; } else { - asm.movptr(temp, src); + asm.movptr(temp, new AMD64Address(receiver, config.hubOffset)); size = AMD64BaseAssembler.OperandSize.QWORD; } before = asm.cmpAndJcc(size, temp, icSpeculatedKlass, ConditionFlag.NotEqual, null, false); diff --git a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/amd64/AMD64HotSpotMacroAssembler.java b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/amd64/AMD64HotSpotMacroAssembler.java index cb5e2b831943..3af9bae71f40 100644 --- a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/amd64/AMD64HotSpotMacroAssembler.java +++ b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/amd64/AMD64HotSpotMacroAssembler.java @@ -29,7 +29,6 @@ import jdk.graal.compiler.asm.Label; import jdk.graal.compiler.asm.amd64.AMD64Address; -import jdk.graal.compiler.asm.amd64.AMD64Assembler; import jdk.graal.compiler.asm.amd64.AMD64MacroAssembler; import jdk.graal.compiler.core.common.CompressEncoding; import jdk.graal.compiler.core.common.NumUtil; @@ -108,25 +107,28 @@ public void verifyOop(Register value, Register tmp, Register tmp2, boolean compr if (!nonNull) { // first null check the value - testAndJcc(compressed ? DWORD : QWORD, value, value, AMD64Assembler.ConditionFlag.Zero, ok, true); + testAndJcc(compressed ? DWORD : QWORD, value, value, ConditionFlag.Zero, ok, true); } - AMD64Address hubAddress; + Register object = value; if (compressed) { CompressEncoding encoding = config.getOopEncoding(); Register heapBaseRegister = AMD64Move.UncompressPointerOp.hasBase(encoding) ? providers.getRegisters().getHeapBaseRegister() : Register.None; movq(tmp, value); AMD64Move.UncompressPointerOp.emitUncompressCode(this, tmp, encoding.getShift(), heapBaseRegister, true); - hubAddress = new AMD64Address(tmp, config.hubOffset); - } else { - hubAddress = new AMD64Address(value, config.hubOffset); + object = tmp; } - // Load the klass + // Load the klass into tmp if (config.useCompressedClassPointers) { - AMD64HotSpotMove.decodeKlassPointer(this, tmp, tmp2, hubAddress, config); + if (config.useCompactObjectHeaders) { + loadCompactClassPointer(tmp, object); + } else { + movl(tmp, new AMD64Address(object, config.hubOffset)); + } + AMD64HotSpotMove.decodeKlassPointer(this, tmp, tmp2, config); } else { - movq(tmp, hubAddress); + movq(tmp, new AMD64Address(object, config.hubOffset)); } // Klass::_super_check_offset movl(tmp2, new AMD64Address(tmp, config.superCheckOffsetOffset)); @@ -136,7 +138,7 @@ public void verifyOop(Register value, Register tmp, Register tmp2, boolean compr // Load the klass from the primary supers movq(tmp2, new AMD64Address(tmp, tmp2, Stride.S1)); // the Klass* should be equal - cmpqAndJcc(tmp2, tmp, AMD64Assembler.ConditionFlag.Equal, ok, true); + cmpqAndJcc(tmp2, tmp, ConditionFlag.Equal, ok, true); illegal(); bind(ok); } @@ -176,4 +178,10 @@ protected final int membarOffset() { public Register getZeroValueRegister() { return providers.getRegisters().getZeroValueRegister(config); } + + public void loadCompactClassPointer(Register result, Register receiver) { + GraalError.guarantee(config.useCompactObjectHeaders, "Load class pointer from markWord only when UseCompactObjectHeaders is on"); + movq(result, new AMD64Address(receiver, config.markOffset)); + shrq(result, config.markWordKlassShift); + } } diff --git a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/amd64/AMD64HotSpotMove.java b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/amd64/AMD64HotSpotMove.java index 09a4bd811feb..aa998cbe3a89 100644 --- a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/amd64/AMD64HotSpotMove.java +++ b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/amd64/AMD64HotSpotMove.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2013, 2021, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2013, 2024, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -169,9 +169,12 @@ public AllocatableValue getResult() { } } - public static void decodeKlassPointer(AMD64MacroAssembler masm, Register register, Register scratch, AMD64Address address, GraalHotSpotVMConfig config) { + /** + * Decode compressed class pointer stored in {@code register}. The content in {@code scratch} + * might be destroyed. + */ + public static void decodeKlassPointer(AMD64MacroAssembler masm, Register register, Register scratch, GraalHotSpotVMConfig config) { CompressEncoding encoding = config.getKlassEncoding(); - masm.movl(register, address); if (encoding.getShift() != 0) { masm.shlq(register, encoding.getShift()); } diff --git a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/meta/DefaultHotSpotLoweringProvider.java b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/meta/DefaultHotSpotLoweringProvider.java index c51b9d5131ab..b4a9ea9a8ada 100644 --- a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/meta/DefaultHotSpotLoweringProvider.java +++ b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/meta/DefaultHotSpotLoweringProvider.java @@ -29,6 +29,11 @@ import static jdk.graal.compiler.core.common.GraalOptions.OmitHotExceptionStacktrace; import static jdk.graal.compiler.hotspot.meta.HotSpotForeignCallsProviderImpl.OSR_MIGRATION_END; import static jdk.graal.compiler.hotspot.meta.HotSpotHostForeignCallsProvider.GENERIC_ARRAYCOPY; +import static jdk.graal.compiler.hotspot.replacements.HotSpotReplacementsUtil.COMPACT_HUB_LOCATION; +import static jdk.graal.compiler.hotspot.replacements.HotSpotReplacementsUtil.COMPRESSED_HUB_LOCATION; +import static jdk.graal.compiler.hotspot.replacements.HotSpotReplacementsUtil.HUB_LOCATION; +import static jdk.graal.compiler.hotspot.replacements.HotSpotReplacementsUtil.HUB_WRITE_LOCATION; +import static jdk.graal.compiler.hotspot.replacements.HotSpotReplacementsUtil.MARK_WORD_LOCATION; import static jdk.vm.ci.services.Services.IS_IN_NATIVE_IMAGE; import static org.graalvm.word.LocationIdentity.any; @@ -103,6 +108,7 @@ import jdk.graal.compiler.hotspot.replacements.arraycopy.HotSpotArraycopySnippets; import jdk.graal.compiler.hotspot.stubs.ForeignCallSnippets; import jdk.graal.compiler.hotspot.word.KlassPointer; +import jdk.graal.compiler.hotspot.word.PointerCastNode; import jdk.graal.compiler.nodes.AbstractBeginNode; import jdk.graal.compiler.nodes.AbstractDeoptimizeNode; import jdk.graal.compiler.nodes.CompressionNode.CompressionOp; @@ -134,11 +140,13 @@ import jdk.graal.compiler.nodes.calc.IntegerDivRemNode; import jdk.graal.compiler.nodes.calc.IsNullNode; import jdk.graal.compiler.nodes.calc.LeftShiftNode; +import jdk.graal.compiler.nodes.calc.NarrowNode; import jdk.graal.compiler.nodes.calc.RemNode; import jdk.graal.compiler.nodes.calc.SignedDivNode; import jdk.graal.compiler.nodes.calc.SignedFloatingIntegerDivNode; import jdk.graal.compiler.nodes.calc.SignedFloatingIntegerRemNode; import jdk.graal.compiler.nodes.calc.SignedRemNode; +import jdk.graal.compiler.nodes.calc.UnsignedRightShiftNode; import jdk.graal.compiler.nodes.debug.VerifyHeapNode; import jdk.graal.compiler.nodes.extended.BranchProbabilityNode; import jdk.graal.compiler.nodes.extended.BytecodeExceptionNode; @@ -356,7 +364,7 @@ public final void initializeExtensions(OptionValues options, Iterable| unused_gap:1 age:4 unused_gap:1 lock:2 (normal object) + * unused:22 hash:31 -->| unused_gap:4 age:4 self-fwd:1 lock:2 (normal object) + * + * 64 bits (with compact headers): + * -------- + * nklass:22 hash:31 -->| unused_gap:4 age:4 self-fwd:1 lock:2 (normal object) * * - hash contains the identity hash value: largest value is * 31 bits, see os::random(). Also, 64-bit vm's require @@ -175,6 +179,7 @@ * [header | 00] locked locked regular object header (fast-locking in use) * [header | 01] unlocked regular object header * [ptr | 10] monitor inflated lock (header is swapped out) + * [header | 10] monitor inflated lock (UseObjectMonitorTable == true) * [ptr | 11] marked used to mark an object * [0 ............ 0| 00] inflating inflation in progress (stack-locking in use) * @@ -557,7 +562,7 @@ private static boolean tryLightweightUnlocking(Object object, Word thread, Word // We elide the monitor check, let the CAS fail instead. // Try to unlock. Transition lock bits 0b00 => 0b01 - Word markLocked = mark.and(~lockMaskInPlace(INJECTED_VMCONFIG)); + Word markLocked = mark.and(WordFactory.unsigned(~markWordLockMaskInPlace(INJECTED_VMCONFIG))); Word markUnlocked = mark.or(unlockedValue(INJECTED_VMCONFIG)); if (probability(FAST_PATH_PROBABILITY, objectPointer.logicCompareAndSwapWord(markOffset(INJECTED_VMCONFIG), markLocked, markUnlocked, MARK_WORD_LOCATION))) { traceObject(trace, "-lock{lightweight:cas}", object, false); diff --git a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/word/HotSpotOperation.java b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/word/HotSpotOperation.java index 3b1c02665350..878f20dd0dd1 100644 --- a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/word/HotSpotOperation.java +++ b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/word/HotSpotOperation.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2014, 2018, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2014, 2024, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -35,6 +35,7 @@ enum HotspotOpcode { FROM_POINTER, + FROM_COMPRESSED_POINTER, TO_KLASS_POINTER, TO_METHOD_POINTER, POINTER_EQ, diff --git a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/word/MetaspacePointer.java b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/word/MetaspacePointer.java index 9a1e4b826022..e4358ccbfa1f 100644 --- a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/word/MetaspacePointer.java +++ b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/word/MetaspacePointer.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2014, 2020, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2014, 2024, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -24,6 +24,7 @@ */ package jdk.graal.compiler.hotspot.word; +import static jdk.graal.compiler.hotspot.word.HotSpotOperation.HotspotOpcode.FROM_COMPRESSED_POINTER; import static jdk.graal.compiler.hotspot.word.HotSpotOperation.HotspotOpcode.FROM_POINTER; import static jdk.graal.compiler.hotspot.word.HotSpotOperation.HotspotOpcode.IS_NULL; @@ -48,6 +49,9 @@ public abstract class MetaspacePointer { @HotSpotOperation(opcode = FROM_POINTER) public abstract Word asWord(); + @HotSpotOperation(opcode = FROM_COMPRESSED_POINTER) + public abstract int asInt(); + /** * Reads the memory at address {@code (this + offset)}. Both the base address and offset are in * bytes. diff --git a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/word/PointerCastNode.java b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/word/PointerCastNode.java index 16c3d4b24e43..a1e4f28415aa 100644 --- a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/word/PointerCastNode.java +++ b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/word/PointerCastNode.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2014, 2022, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2014, 2024, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -30,21 +30,21 @@ import jdk.graal.compiler.core.common.type.Stamp; import jdk.graal.compiler.graph.Node; import jdk.graal.compiler.graph.NodeClass; -import jdk.graal.compiler.nodes.spi.Canonicalizable; -import jdk.graal.compiler.nodes.spi.CanonicalizerTool; import jdk.graal.compiler.hotspot.word.HotSpotOperation.HotspotOpcode; import jdk.graal.compiler.nodeinfo.NodeInfo; import jdk.graal.compiler.nodes.NodeView; import jdk.graal.compiler.nodes.ValueNode; import jdk.graal.compiler.nodes.calc.FloatingNode; +import jdk.graal.compiler.nodes.spi.Canonicalizable; +import jdk.graal.compiler.nodes.spi.CanonicalizerTool; import jdk.graal.compiler.nodes.spi.LIRLowerable; import jdk.graal.compiler.nodes.spi.NodeLIRBuilderTool; - import jdk.vm.ci.meta.Value; /** - * Cast between Word and metaspace pointers exposed by the {@link HotspotOpcode#FROM_POINTER} and - * {@link HotspotOpcode#TO_KLASS_POINTER} operations. + * Cast between Word and metaspace pointers exposed by the {@link HotspotOpcode#FROM_POINTER}, + * {@link HotspotOpcode#FROM_COMPRESSED_POINTER} and {@link HotspotOpcode#TO_KLASS_POINTER} + * operations. */ @NodeInfo(cycles = CYCLES_0, size = SIZE_0) public final class PointerCastNode extends FloatingNode implements Canonicalizable, LIRLowerable, Node.ValueNumberable { diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/util/IdentityHashCodeUtil.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/util/IdentityHashCodeUtil.java index 2075875b49b5..8fa1b77b6bc7 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/util/IdentityHashCodeUtil.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/util/IdentityHashCodeUtil.java @@ -52,7 +52,7 @@ public class IdentityHashCodeUtil { static { unsafe = Unsafe.getUnsafe(); config = GraalAccess.getGraalCapability(GraalHotSpotVMConfig.class); - hashCodeMask = NumUtil.getNbitNumberLong(31) << config.identityHashCodeShift; + hashCodeMask = NumUtil.getNbitNumberLong(31) << config.markWordHashCodeShift; } /** @@ -116,7 +116,7 @@ private static int readIdentityHashCode(long markWord) { /* * See HotSpotHashCodeSnippets for explanation. */ - long lockBits = markWord & config.lockMaskInPlace; + long lockBits = markWord & config.markWordLockMaskInPlace; boolean containsHashCode; if (config.lockingMode == config.lockingModeLightweight) { containsHashCode = lockBits != config.monitorValue; @@ -124,7 +124,7 @@ private static int readIdentityHashCode(long markWord) { containsHashCode = lockBits == config.unlockedValue; } if (containsHashCode) { - int hashcode = (int) (markWord >>> config.identityHashCodeShift); + int hashcode = (int) ((markWord & hashCodeMask) >>> config.markWordHashCodeShift); if (hashcode == config.uninitializedIdentityHashCodeValue) { return UNINITIALIZED; } @@ -138,11 +138,12 @@ private static int readIdentityHashCode(long markWord) { * @return true if the object's identity hash code was set. */ private static boolean trySetIdentityHashCode(Object obj, long originalMarkWord, int hashCode) { + long hashInPlace = (((long) hashCode) << config.markWordHashCodeShift) & hashCodeMask; long newMarkWord; if (config.uninitializedIdentityHashCodeValue == 0) { - newMarkWord = originalMarkWord | (((long) hashCode) << config.identityHashCodeShift); + newMarkWord = originalMarkWord | hashInPlace; } else { - newMarkWord = (originalMarkWord & (~hashCodeMask)) | (((long) hashCode) << config.identityHashCodeShift); + newMarkWord = (originalMarkWord & (~hashCodeMask)) | hashInPlace; } return unsafe.compareAndSetLong(obj, config.markOffset, originalMarkWord, newMarkWord); }