From 7df90dfd83be7bba1c248ccdc12e060bfa31367c Mon Sep 17 00:00:00 2001 From: Jatin Bhateja Date: Wed, 22 Nov 2023 13:04:50 +0000 Subject: [PATCH] 8319972: [lworld+vector] Enable intrinsification of Unsafe.finishPrivateBuffer. Reviewed-by: xgong --- src/hotspot/share/opto/graphKit.cpp | 13 ++++- src/hotspot/share/opto/inlinetypenode.cpp | 48 ++++++------------ src/hotspot/share/opto/inlinetypenode.hpp | 4 +- src/hotspot/share/opto/library_call.cpp | 49 +++++++++--------- src/hotspot/share/opto/macro.cpp | 5 +- src/hotspot/share/opto/parse1.cpp | 13 +++-- .../jdk/incubator/vector/AbstractMask.java | 50 ++++++++++--------- .../jdk/incubator/vector/AbstractShuffle.java | 28 ++++++----- .../jdk/incubator/vector/AbstractVector.java | 3 ++ .../jdk/incubator/vector/ByteVector.java | 2 - .../jdk/incubator/vector/DoubleVector.java | 2 - .../jdk/incubator/vector/FloatVector.java | 2 - .../jdk/incubator/vector/IntVector.java | 2 - .../jdk/incubator/vector/LongVector.java | 2 - .../jdk/incubator/vector/ShortVector.java | 2 - .../incubator/vector/X-Vector.java.template | 2 - test/hotspot/jtreg/ProblemList.txt | 1 + .../valhalla/inlinetypes/TestIntrinsics.java | 2 +- 18 files changed, 109 insertions(+), 121 deletions(-) diff --git a/src/hotspot/share/opto/graphKit.cpp b/src/hotspot/share/opto/graphKit.cpp index 1802d9262b2..673489ac2c3 100644 --- a/src/hotspot/share/opto/graphKit.cpp +++ b/src/hotspot/share/opto/graphKit.cpp @@ -54,6 +54,7 @@ #include "utilities/bitMap.inline.hpp" #include "utilities/powerOfTwo.hpp" #include "utilities/growableArray.hpp" +#include "classfile/vmSymbols.hpp" //----------------------------GraphKit----------------------------------------- // Main utility constructor. @@ -1863,7 +1864,6 @@ void GraphKit::set_arguments_for_java_call(CallJavaNode* call, bool is_late_inli if (t->is_inlinetypeptr() && !call->method()->get_Method()->mismatch() && call->method()->is_scalarized_arg(arg_num)) { // We don't pass inline type arguments by reference but instead pass each field of the inline type if (!arg->is_InlineType()) { - assert(_gvn.type(arg)->is_zero_type() && !t->inline_klass()->is_null_free(), "Unexpected argument type"); arg = InlineTypeNode::make_from_oop(this, arg, t->inline_klass(), t->inline_klass()->is_null_free()); } InlineTypeNode* vt = arg->as_InlineType(); @@ -1956,6 +1956,15 @@ Node* GraphKit::set_results_for_java_call(CallJavaNode* call, bool separate_io_p if (type->is_inlinetypeptr()) { ret = InlineTypeNode::make_from_oop(this, ret, type->inline_klass(), type->inline_klass()->is_null_free()); } + Node* receiver = !call->method()->is_static() ? call->in(TypeFunc::Parms) : nullptr; + if (ret->is_InlineType() && + receiver && receiver->bottom_type()->isa_instptr() && + receiver->bottom_type()->is_instptr()->instance_klass()->name()->get_symbol() == vmSymbols::jdk_internal_misc_Unsafe() && + call->method()->name()->get_symbol() == vmSymbols::makePrivateBuffer_name()) { + // Re-buffer scalarized InlineTypeNodes returned from makePrivateBuffer + // and transition the allocation into larval state. + ret = ret->as_InlineType()->make_larval(this); + } } } @@ -3416,7 +3425,7 @@ Node* GraphKit::gen_checkcast(Node *obj, Node* superklass, Node* *failure_contro assert(safe_for_replace, "must be"); obj = null_check(obj); } - assert(stopped() || !toop->is_inlinetypeptr() || obj->is_InlineType(), "should have been scalarized"); + assert(stopped() || !toop->is_inlinetypeptr() || obj->is_InlineType() || obj->bottom_type()->is_inlinetypeptr(), "should have been scalarized"); return obj; case Compile::SSC_always_false: if (null_free) { diff --git a/src/hotspot/share/opto/inlinetypenode.cpp b/src/hotspot/share/opto/inlinetypenode.cpp index d9c0ceeb02d..b6e62c85a50 100644 --- a/src/hotspot/share/opto/inlinetypenode.cpp +++ b/src/hotspot/share/opto/inlinetypenode.cpp @@ -797,7 +797,7 @@ Node* InlineTypeNode::Ideal(PhaseGVN* phase, bool can_reshape) { // but later decide to inline the call after the callee code also triggered allocation. for (DUIterator_Fast imax, i = fast_outs(imax); i < imax; i++) { AllocateNode* alloc = fast_out(i)->isa_Allocate(); - if (alloc != nullptr && alloc->in(AllocateNode::InlineType) == this && !alloc->_is_scalar_replaceable) { + if (alloc != nullptr && alloc->in(AllocateNode::InlineType) == this && !alloc->_is_scalar_replaceable && !alloc->_larval) { // Found a re-allocation Node* res = alloc->result_cast(); if (res != nullptr && res->is_CheckCastPP()) { @@ -1009,34 +1009,22 @@ InlineTypeNode* InlineTypeNode::make_from_multi(GraphKit* kit, MultiNode* multi, return kit->gvn().transform(vt)->as_InlineType(); } -InlineTypeNode* InlineTypeNode::make_larval(GraphKit* kit, bool allocate) const { +Node* InlineTypeNode::make_larval(GraphKit* kit) const { ciInlineKlass* vk = inline_klass(); - InlineTypeNode* res = make_uninitialized(kit->gvn(), vk); - for (uint i = 1; i < req(); ++i) { - res->set_req(i, in(i)); - } - if (allocate) { - // Re-execute if buffering triggers deoptimization - PreserveReexecuteState preexecs(kit); - kit->jvms()->set_should_reexecute(true); - Node* klass_node = kit->makecon(TypeKlassPtr::make(vk)); - Node* alloc_oop = kit->new_instance(klass_node, nullptr, nullptr, true); - AllocateNode* alloc = AllocateNode::Ideal_allocation(alloc_oop); - alloc->_larval = true; + // Re-execute if buffering triggers deoptimization + PreserveReexecuteState preexecs(kit); + kit->jvms()->set_should_reexecute(true); + Node* klass_node = kit->makecon(TypeKlassPtr::make(vk)); + Node* alloc_oop = kit->new_instance(klass_node, nullptr, nullptr, true); + AllocateNode* alloc = AllocateNode::Ideal_allocation(alloc_oop); + alloc->_larval = true; - store(kit, alloc_oop, alloc_oop, vk); - res->set_oop(alloc_oop); - } - // TODO 8239003 - //res->set_type(TypeInlineType::make(vk, true)); - res = kit->gvn().transform(res)->as_InlineType(); - assert(!allocate || res->is_allocated(&kit->gvn()), "must be allocated"); - return res; + store(kit, alloc_oop, alloc_oop, vk); + return alloc_oop; } -InlineTypeNode* InlineTypeNode::finish_larval(GraphKit* kit) const { - Node* obj = get_oop(); +InlineTypeNode* InlineTypeNode::finish_larval(GraphKit* kit, Node* obj, const TypeInstPtr* vk) { Node* mark_addr = kit->basic_plus_adr(obj, oopDesc::mark_offset_in_bytes()); Node* mark = kit->make_load(nullptr, mark_addr, TypeX_X, TypeX_X->basic_type(), MemNode::unordered); mark = kit->gvn().transform(new AndXNode(mark, kit->MakeConX(~markWord::larval_bit_in_place))); @@ -1048,15 +1036,7 @@ InlineTypeNode* InlineTypeNode::finish_larval(GraphKit* kit) const { assert(alloc != nullptr, "must have an allocation node"); kit->insert_mem_bar(Op_MemBarStoreStore, alloc->proj_out_or_null(AllocateNode::RawAddress)); - ciInlineKlass* vk = inline_klass(); - InlineTypeNode* res = make_uninitialized(kit->gvn(), vk); - for (uint i = 1; i < req(); ++i) { - res->set_req(i, in(i)); - } - // TODO 8239003 - //res->set_type(TypeInlineType::make(vk, false)); - res = kit->gvn().transform(res)->as_InlineType(); - return res; + return InlineTypeNode::make_from_oop(kit, obj, vk->inline_klass(), !vk->maybe_null()); } bool InlineTypeNode::is_larval(PhaseGVN* gvn) const { @@ -1281,7 +1261,7 @@ void InlineTypeNode::remove_redundant_allocations(PhaseIdealLoop* phase) { // will be removed anyway and changing the memory chain will confuse other optimizations. for (DUIterator_Fast imax, i = fast_outs(imax); i < imax; i++) { AllocateNode* alloc = fast_out(i)->isa_Allocate(); - if (alloc != nullptr && alloc->in(AllocateNode::InlineType) == this && !alloc->_is_scalar_replaceable) { + if (alloc != nullptr && alloc->in(AllocateNode::InlineType) == this && !alloc->_is_scalar_replaceable && !alloc->_larval) { Node* res = alloc->result_cast(); if (res == nullptr || !res->is_CheckCastPP()) { break; // No unique CheckCastPP diff --git a/src/hotspot/share/opto/inlinetypenode.hpp b/src/hotspot/share/opto/inlinetypenode.hpp index 1cea4925ddd..2a575724746 100644 --- a/src/hotspot/share/opto/inlinetypenode.hpp +++ b/src/hotspot/share/opto/inlinetypenode.hpp @@ -164,8 +164,8 @@ class InlineTypeNode : public TypeNode { // Pass inline type as fields at a call or return void pass_fields(GraphKit* kit, Node* n, uint& base_input, bool in, bool null_free = true); - InlineTypeNode* make_larval(GraphKit* kit, bool allocate) const; - InlineTypeNode* finish_larval(GraphKit* kit) const; + Node* make_larval(GraphKit* kit) const; + static InlineTypeNode* finish_larval(GraphKit* kit, Node* obj, const TypeInstPtr* vk); // Allocation optimizations void remove_redundant_allocations(PhaseIdealLoop* phase); diff --git a/src/hotspot/share/opto/library_call.cpp b/src/hotspot/share/opto/library_call.cpp index 9bc36d2144e..ce3c060d9ab 100644 --- a/src/hotspot/share/opto/library_call.cpp +++ b/src/hotspot/share/opto/library_call.cpp @@ -2329,6 +2329,17 @@ bool LibraryCallKit::inline_unsafe_access(bool is_store, const BasicType type, c assert(Unsafe_field_offset_to_byte_offset(11) == 11, "fieldOffset must be byte-scaled"); + if (_gvn.type(base)->is_inlinetypeptr() && is_store) { + // FIXME: Larval bit check is needed to preserve the semantics of value + // objects which can be mutated only if its _larval bit is set. Since + // the oop is not always an AllocateNode, we have to find an utility way + // to check the larval state for all kind of oops. + AllocateNode* alloc = AllocateNode::Ideal_allocation(base); + if (alloc != nullptr) { + assert(alloc->_larval, "InlineType instance must be in _larval state for unsafe put operation.\n"); + } + } + ciInlineKlass* inline_klass = nullptr; if (type == T_PRIMITIVE_OBJECT) { const TypeInstPtr* cls = _gvn.type(argument(4))->isa_instptr(); @@ -2349,14 +2360,6 @@ bool LibraryCallKit::inline_unsafe_access(bool is_store, const BasicType type, c return false; } base = vt->get_oop(); - // FIXME: Larval bit check is needed to preserve the semantics of value - // objects which can be mutated only if its _larval bit is set. Since - // the oop is not always an AllocateNode, we have to find an utility way - // to check the larval state for all kind of oops. - AllocateNode* alloc = AllocateNode::Ideal_allocation(base); - if (alloc != nullptr) { - assert(alloc->_larval, "InlineType instance must be in _larval state for unsafe put operation.\n"); - } } else { if (offset->is_Con()) { long off = find_long_con(offset, 0); @@ -2627,11 +2630,6 @@ bool LibraryCallKit::inline_unsafe_access(bool is_store, const BasicType type, c } } - if (argument(1)->is_InlineType() && is_store) { - InlineTypeNode* value = InlineTypeNode::make_from_oop(this, base, _gvn.type(argument(1))->inline_klass()); - value = value->make_larval(this, false); - replace_in_map(argument(1), value); - } return true; } @@ -2648,31 +2646,32 @@ bool LibraryCallKit::inline_unsafe_make_private_buffer() { return true; } - set_result(value->as_InlineType()->make_larval(this, true)); + set_result(value->as_InlineType()->make_larval(this)); return true; } bool LibraryCallKit::inline_unsafe_finish_private_buffer() { Node* receiver = argument(0); Node* buffer = argument(1); - if (!buffer->is_InlineType()) { - return false; - } - InlineTypeNode* vt = buffer->as_InlineType(); - if (!vt->is_allocated(&_gvn) || VectorSupport::is_vector_payload_mf(vt->inline_klass()->get_InlineKlass())) { - return false; + + // Incoming value should be a buffer with inline type and not InlineTypeNode. + if (buffer->is_InlineType() || !buffer->bottom_type()->is_inlinetypeptr()) { + return false; } - // TODO 8239003 Why is this needed? - if (AllocateNode::Ideal_allocation(vt->get_oop()) == nullptr) { + + // Allocation node must exist to generate IR for transitioning allocation out + // of larval state. Disable the intrinsic and take unsafe slow path if allocation + // is not reachable, oop returned by Unsafe_finishPrivateBuffer native method + // will automatically rematerialize InlineTypeNode. + if (AllocateNode::Ideal_allocation(buffer) == nullptr) { return false; } - receiver = null_check(receiver); if (stopped()) { return true; } - - set_result(vt->finish_larval(this)); + const TypeInstPtr* ptr = buffer->bottom_type()->isa_instptr(); + set_result(InlineTypeNode::finish_larval(this, buffer, ptr)); return true; } diff --git a/src/hotspot/share/opto/macro.cpp b/src/hotspot/share/opto/macro.cpp index d5b993323f3..82a9df55b00 100644 --- a/src/hotspot/share/opto/macro.cpp +++ b/src/hotspot/share/opto/macro.cpp @@ -958,7 +958,10 @@ bool PhaseMacroExpand::scalar_replacement(AllocateNode *alloc, GrowableArray isa_oopptr(); } - + // Bufferes in larval state should not be scalarized. + if (alloc->_larval) { + return false; + } // Process the safepoint uses assert(safepoints.length() == 0 || !res_type->is_inlinetypeptr(), "Inline type allocations should not have safepoint uses"); Unique_Node_List value_worklist; diff --git a/src/hotspot/share/opto/parse1.cpp b/src/hotspot/share/opto/parse1.cpp index c553847aa31..f1294a4e4ab 100644 --- a/src/hotspot/share/opto/parse1.cpp +++ b/src/hotspot/share/opto/parse1.cpp @@ -1777,9 +1777,12 @@ void Parse::merge_common(Parse::Block* target, int pnum) { // Allocate inline type in src block to be able to merge it with oop in target block map()->set_req(j, n->as_InlineType()->buffer(this)); } else if (!n->is_InlineType() && t->is_inlinetypeptr()) { - // Scalarize null in src block to be able to merge it with inline type in target block - assert(gvn().type(n)->is_zero_type(), "Should have been scalarized"); - map()->set_req(j, InlineTypeNode::make_null(gvn(), t->inline_klass())); + AllocateNode* alloc = AllocateNode::Ideal_allocation(n); + if (alloc == nullptr || !alloc->_larval) { + // Scalarize null in src block to be able to merge it with inline type in target block + assert(gvn().type(n)->is_zero_type(), "Should have been scalarized"); + map()->set_req(j, InlineTypeNode::make_null(gvn(), t->inline_klass())); + } } } } @@ -1884,7 +1887,7 @@ void Parse::merge_common(Parse::Block* target, int pnum) { PhiNode* phi; if (m->is_Phi() && m->as_Phi()->region() == r) { phi = m->as_Phi(); - } else if (m->is_InlineType() && m->as_InlineType()->has_phi_inputs(r)) { + } else if (m->is_InlineType() && n->is_InlineType() && m->as_InlineType()->has_phi_inputs(r)) { phi = m->as_InlineType()->get_oop()->as_Phi(); } else { phi = nullptr; @@ -1923,7 +1926,7 @@ void Parse::merge_common(Parse::Block* target, int pnum) { // It is a bug if we create a phi which sees a garbage value on a live path. // Merging two inline types? - if (phi != nullptr && phi->bottom_type()->is_inlinetypeptr()) { + if (phi != nullptr && phi->bottom_type()->is_inlinetypeptr() && m->is_InlineType() && n->is_InlineType()) { // Reload current state because it may have been updated by ensure_phi m = map()->in(j); InlineTypeNode* vtm = m->as_InlineType(); // Current inline type diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/AbstractMask.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/AbstractMask.java index 1097feeb19d..bc1b09b9790 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/AbstractMask.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/AbstractMask.java @@ -38,31 +38,33 @@ abstract class AbstractMask extends VectorMask { + public static final Unsafe U = Unsafe.getUnsafe(); + /*package-private*/ abstract VectorPayloadMF getBits(); static VectorPayloadMF prepare(VectorPayloadMF payload, int offset, VectorSpecies species) { boolean isMaxShape = species.vectorShape() == VectorShape.S_Max_BIT; VectorPayloadMF res = VectorPayloadMF.newMaskInstanceFactory(species.elementType(), species.length(), isMaxShape); - res = Unsafe.getUnsafe().makePrivateBuffer(res); + res = U.makePrivateBuffer(res); long mOffset = res.multiFieldOffset(); for (int i = 0; i < species.length(); i++) { - boolean b = Unsafe.getUnsafe().getBoolean(payload, mOffset + i + offset); - Unsafe.getUnsafe().putBoolean(res, mOffset + i, b); + boolean b = U.getBoolean(payload, mOffset + i + offset); + U.putBoolean(res, mOffset + i, b); } - res = Unsafe.getUnsafe().finishPrivateBuffer(res); + res = U.finishPrivateBuffer(res); return res; } static VectorPayloadMF prepare(boolean val, VectorSpecies species) { boolean isMaxShape = species.vectorShape() == VectorShape.S_Max_BIT; VectorPayloadMF res = VectorPayloadMF.newMaskInstanceFactory(species.elementType(), species.length(), isMaxShape); - res = Unsafe.getUnsafe().makePrivateBuffer(res); + res = U.makePrivateBuffer(res); long mOffset = res.multiFieldOffset(); for (int i = 0; i < species.length(); i++) { - Unsafe.getUnsafe().putBoolean(res, mOffset + i, val); + U.putBoolean(res, mOffset + i, val); } - res = Unsafe.getUnsafe().finishPrivateBuffer(res); + res = U.finishPrivateBuffer(res); return res; } @@ -77,13 +79,13 @@ AbstractMask uOpMF(MUnOp f) { VectorPayloadMF bits = getBits(); boolean is_max_species = ((AbstractSpecies)(vspecies())).is_max_species(); VectorPayloadMF res = VectorPayloadMF.newMaskInstanceFactory(vspecies().elementType(), length, is_max_species); - res = Unsafe.getUnsafe().makePrivateBuffer(res); + res = U.makePrivateBuffer(res); long mOffset = res.multiFieldOffset(); for (int i = 0; i < length; i++) { - boolean b = Unsafe.getUnsafe().getBoolean(bits, mOffset + i); - Unsafe.getUnsafe().putBoolean(res, mOffset + i, f.apply(i, b)); + boolean b = U.getBoolean(bits, mOffset + i); + U.putBoolean(res, mOffset + i, f.apply(i, b)); } - res = Unsafe.getUnsafe().finishPrivateBuffer(res); + res = U.finishPrivateBuffer(res); return vspecies().maskFactory(res); } @@ -99,14 +101,14 @@ AbstractMask bOpMF(AbstractMask m, MBinOp f) { VectorPayloadMF mbits = m.getBits(); boolean is_max_species = ((AbstractSpecies)(vspecies())).is_max_species(); VectorPayloadMF res = VectorPayloadMF.newMaskInstanceFactory(vspecies().elementType(), length, is_max_species); - res = Unsafe.getUnsafe().makePrivateBuffer(res); + res = U.makePrivateBuffer(res); long mOffset = res.multiFieldOffset(); for (int i = 0; i < length; i++) { - boolean b = Unsafe.getUnsafe().getBoolean(bits, mOffset + i); - boolean mb = Unsafe.getUnsafe().getBoolean(mbits, mOffset + i); - Unsafe.getUnsafe().putBoolean(res, mOffset + i, f.apply(i, b, mb)); + boolean b = U.getBoolean(bits, mOffset + i); + boolean mb = U.getBoolean(mbits, mOffset + i); + U.putBoolean(res, mOffset + i, f.apply(i, b, mb)); } - res = Unsafe.getUnsafe().finishPrivateBuffer(res); + res = U.finishPrivateBuffer(res); return vspecies().maskFactory(res); } @@ -116,7 +118,7 @@ void stOpMF(boolean[] arr, int idx) { VectorPayloadMF bits = getBits(); long mOffset = bits.multiFieldOffset(); for (int i = 0; i < vspecies().laneCount(); i++) { - arr[idx++] = Unsafe.getUnsafe().getBoolean(bits, mOffset + i); + arr[idx++] = U.getBoolean(bits, mOffset + i); } } @@ -216,7 +218,7 @@ public final VectorMask eq(VectorMask m) { /*package-private*/ boolean laneIsSetHelper(int idx) { VectorPayloadMF bits = getBits(); - return Unsafe.getUnsafe().getBoolean(bits, bits.multiFieldOffset() + idx); + return U.getBoolean(bits, bits.multiFieldOffset() + idx); } /*package-private*/ @@ -225,7 +227,7 @@ boolean anyTrueHelper() { VectorPayloadMF bits = getBits(); long mOffset = bits.multiFieldOffset(); for (int i = 0; i < length(); i++) { - if (Unsafe.getUnsafe().getBoolean(bits, mOffset + i)) return true; + if (U.getBoolean(bits, mOffset + i)) return true; } return false; } @@ -236,7 +238,7 @@ boolean allTrueHelper() { VectorPayloadMF bits = getBits(); long mOffset = bits.multiFieldOffset(); for (int i = 0; i < length(); i++) { - if (!Unsafe.getUnsafe().getBoolean(bits, mOffset + i)) return false; + if (!U.getBoolean(bits, mOffset + i)) return false; } return true; } @@ -247,7 +249,7 @@ int trueCountHelper() { VectorPayloadMF bits = getBits(); long mOffset = bits.multiFieldOffset(); for (int i = 0; i < length(); i++) { - if (Unsafe.getUnsafe().getBoolean(bits, mOffset + i)) c++; + if (U.getBoolean(bits, mOffset + i)) c++; } return c; } @@ -257,7 +259,7 @@ int firstTrueHelper() { VectorPayloadMF bits = getBits(); long mOffset = bits.multiFieldOffset(); for (int i = 0; i < length(); i++) { - if (Unsafe.getUnsafe().getBoolean(bits, mOffset + i)) return i; + if (U.getBoolean(bits, mOffset + i)) return i; } return length(); } @@ -267,7 +269,7 @@ int lastTrueHelper() { VectorPayloadMF bits = getBits(); long mOffset = bits.multiFieldOffset(); for (int i = length() - 1; i >= 0; i--) { - if (Unsafe.getUnsafe().getBoolean(bits, mOffset + i)) return i; + if (U.getBoolean(bits, mOffset + i)) return i; } return -1; } @@ -279,7 +281,7 @@ long toLongHelper() { VectorPayloadMF bits = getBits(); long mOffset = bits.multiFieldOffset(); for (int i = 0; i < length(); i++) { - res = Unsafe.getUnsafe().getBoolean(bits, mOffset + i) ? res | set : res; + res = U.getBoolean(bits, mOffset + i) ? res | set : res; set = set << 1; } return res; diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/AbstractShuffle.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/AbstractShuffle.java index 8604d49851a..48bc86199e9 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/AbstractShuffle.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/AbstractShuffle.java @@ -33,6 +33,8 @@ abstract class AbstractShuffle extends VectorShuffle { static final IntUnaryOperator IDENTITY = i -> i; + public static final Unsafe U = Unsafe.getUnsafe(); + // Internal representation allows for a maximum index of 256 // Values are clipped to [-VLENGTH..VLENGTH-1]. @@ -43,14 +45,14 @@ static VectorPayloadMF prepare(int[] indices, int offset, VectorSpecies s int length = species.length(); boolean isMaxShape = species.vectorShape() == VectorShape.S_Max_BIT; VectorPayloadMF payload = VectorPayloadMF.newShuffleInstanceFactory(species.elementType(), length, isMaxShape); - payload = Unsafe.getUnsafe().makePrivateBuffer(payload); + payload = U.makePrivateBuffer(payload); long mf_offset = payload.multiFieldOffset(); for (int i = 0; i < length; i++) { int si = indices[offset + i]; si = partiallyWrapIndex(si, length); - Unsafe.getUnsafe().putByte(payload, mf_offset + i * Byte.BYTES, (byte) si); + U.putByte(payload, mf_offset + i * Byte.BYTES, (byte) si); } - payload = Unsafe.getUnsafe().finishPrivateBuffer(payload); + payload = U.finishPrivateBuffer(payload); return payload; } @@ -58,14 +60,14 @@ static VectorPayloadMF prepare(IntUnaryOperator f, VectorSpecies species) int length = species.length(); boolean isMaxShape = species.vectorShape() == VectorShape.S_Max_BIT; VectorPayloadMF payload = VectorPayloadMF.newShuffleInstanceFactory(species.elementType(), length, isMaxShape); - payload = Unsafe.getUnsafe().makePrivateBuffer(payload); + payload = U.makePrivateBuffer(payload); long offset = payload.multiFieldOffset(); for (int i = 0; i < length; i++) { int si = f.applyAsInt(i); si = partiallyWrapIndex(si, length); - Unsafe.getUnsafe().putByte(payload, offset + i * Byte.BYTES, (byte) si); + U.putByte(payload, offset + i * Byte.BYTES, (byte) si); } - payload = Unsafe.getUnsafe().finishPrivateBuffer(payload); + payload = U.finishPrivateBuffer(payload); return payload; } @@ -85,7 +87,7 @@ public void intoArray(int[] a, int offset) { int vlen = indices.length(); long mf_offset = indices.multiFieldOffset(); for (int i = 0; i < vlen; i++) { - int sourceIndex = Unsafe.getUnsafe().getByte(indices, mf_offset + i * Byte.BYTES); + int sourceIndex = U.getByte(indices, mf_offset + i * Byte.BYTES); assert(sourceIndex >= -vlen && sourceIndex < vlen); a[offset + i] = sourceIndex; } @@ -123,7 +125,7 @@ public final VectorShuffle checkIndexes() { if (vecmask.anyTrue()) { VectorPayloadMF indices = indices(); long offset = indices.multiFieldOffset(); - throw checkIndexFailed(Unsafe.getUnsafe().getByte(indices, offset + vecmask.firstTrue() * Byte.BYTES), length()); + throw checkIndexFailed(U.getByte(indices, offset + vecmask.firstTrue() * Byte.BYTES), length()); } return this; } @@ -146,9 +148,9 @@ public final VectorShuffle wrapAndRebuild(VectorPayloadMF oldIndices) { boolean is_max_species = ((AbstractSpecies)(vspecies())).is_max_species(); VectorPayloadMF indices = VectorPayloadMF.newShuffleInstanceFactory(vspecies().elementType(), length, is_max_species); long offset = oldIndices.multiFieldOffset(); - indices = Unsafe.getUnsafe().makePrivateBuffer(indices); + indices = U.makePrivateBuffer(indices); for (int i = 0; i < length; i++) { - int si = Unsafe.getUnsafe().getByte(oldIndices, offset + i * Byte.BYTES); + int si = U.getByte(oldIndices, offset + i * Byte.BYTES); // FIXME: This does not work unless it's a power of 2. if ((length & (length - 1)) == 0) { si += si & length; // power-of-two optimization @@ -157,9 +159,9 @@ public final VectorShuffle wrapAndRebuild(VectorPayloadMF oldIndices) { si += length; } assert(si >= 0 && si < length); - Unsafe.getUnsafe().putByte(indices, offset + i * Byte.BYTES, (byte) si); + U.putByte(indices, offset + i * Byte.BYTES, (byte) si); } - indices = Unsafe.getUnsafe().finishPrivateBuffer(indices); + indices = U.finishPrivateBuffer(indices); return vspecies().dummyVectorMF().shuffleFromBytes(indices); } @@ -225,7 +227,7 @@ static boolean indexesInRange(VectorPayloadMF indices) { int length = indices.length(); long offset = indices.multiFieldOffset(); for (int i = 0; i < length; i++) { - byte si = Unsafe.getUnsafe().getByte(indices, offset + i * Byte.BYTES); + byte si = U.getByte(indices, offset + i * Byte.BYTES); if (si >= length || si < -length) { boolean assertsEnabled = false; assert(assertsEnabled = true); diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/AbstractVector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/AbstractVector.java index fe0f303ffaa..de6bdbeeb1c 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/AbstractVector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/AbstractVector.java @@ -30,6 +30,7 @@ import java.nio.ByteOrder; import java.util.function.IntUnaryOperator; +import jdk.internal.misc.Unsafe; import static jdk.incubator.vector.VectorOperators.*; import static jdk.internal.vm.vector.VectorSupport.*; @@ -80,6 +81,8 @@ abstract class AbstractVector extends Vector { /*package-private*/ abstract long multiFieldOffset(); + static final Unsafe U = Unsafe.getUnsafe(); + @Override @ForceInline public final VectorSpecies species() { diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/ByteVector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/ByteVector.java index 0ee74d9dd90..9f8da4c27ea 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/ByteVector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/ByteVector.java @@ -57,8 +57,6 @@ public abstract class ByteVector extends AbstractVector { */ public ByteVector() {} - static final Unsafe U = Unsafe.getUnsafe(); - static final ValueLayout.OfByte ELEMENT_LAYOUT = ValueLayout.JAVA_BYTE.withByteAlignment(1); @ForceInline diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/DoubleVector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/DoubleVector.java index c4fd6bca118..4d880c976e1 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/DoubleVector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/DoubleVector.java @@ -57,8 +57,6 @@ public abstract class DoubleVector extends AbstractVector { */ public DoubleVector() {} - static final Unsafe U = Unsafe.getUnsafe(); - static final ValueLayout.OfDouble ELEMENT_LAYOUT = ValueLayout.JAVA_DOUBLE.withByteAlignment(1); @ForceInline diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/FloatVector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/FloatVector.java index 8bfb408d404..f294926974c 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/FloatVector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/FloatVector.java @@ -57,8 +57,6 @@ public abstract class FloatVector extends AbstractVector { */ public FloatVector() {} - static final Unsafe U = Unsafe.getUnsafe(); - static final ValueLayout.OfFloat ELEMENT_LAYOUT = ValueLayout.JAVA_FLOAT.withByteAlignment(1); @ForceInline diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/IntVector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/IntVector.java index a51b7c341cc..9999335f89d 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/IntVector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/IntVector.java @@ -57,8 +57,6 @@ public abstract class IntVector extends AbstractVector { */ public IntVector() {} - static final Unsafe U = Unsafe.getUnsafe(); - static final ValueLayout.OfInt ELEMENT_LAYOUT = ValueLayout.JAVA_INT.withByteAlignment(1); @ForceInline diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/LongVector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/LongVector.java index 62eb51827ed..13b78694dbe 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/LongVector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/LongVector.java @@ -57,8 +57,6 @@ public abstract class LongVector extends AbstractVector { */ public LongVector() {} - static final Unsafe U = Unsafe.getUnsafe(); - static final ValueLayout.OfLong ELEMENT_LAYOUT = ValueLayout.JAVA_LONG.withByteAlignment(1); @ForceInline diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/ShortVector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/ShortVector.java index a5a12caa62e..0b3a001fe1f 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/ShortVector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/ShortVector.java @@ -57,8 +57,6 @@ public abstract class ShortVector extends AbstractVector { */ public ShortVector() {} - static final Unsafe U = Unsafe.getUnsafe(); - static final ValueLayout.OfShort ELEMENT_LAYOUT = ValueLayout.JAVA_SHORT.withByteAlignment(1); @ForceInline diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/X-Vector.java.template b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/X-Vector.java.template index ef9352f53e1..4f13a8ac15d 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/X-Vector.java.template +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/X-Vector.java.template @@ -61,8 +61,6 @@ public abstract class $abstractvectortype$ extends AbstractVector<$Boxtype$> { */ public $abstractvectortype$() {} - static final Unsafe U = Unsafe.getUnsafe(); - static final ValueLayout.Of$Type$ ELEMENT_LAYOUT = ValueLayout.JAVA_$TYPE$.withByteAlignment(1); @ForceInline diff --git a/test/hotspot/jtreg/ProblemList.txt b/test/hotspot/jtreg/ProblemList.txt index 7d29dfd27b7..8b3515adcf7 100644 --- a/test/hotspot/jtreg/ProblemList.txt +++ b/test/hotspot/jtreg/ProblemList.txt @@ -213,3 +213,4 @@ vmTestbase/nsk/monitoring/ThreadMXBean/ThreadInfo/Multi/Multi005/TestDescription # lworld+vector compiler/vectorapi/VectorReinterpretTest.java +compiler/vectorapi/reshape/TestVectorReinterpret.java diff --git a/test/hotspot/jtreg/compiler/valhalla/inlinetypes/TestIntrinsics.java b/test/hotspot/jtreg/compiler/valhalla/inlinetypes/TestIntrinsics.java index dd627142f05..6d3cd961e62 100644 --- a/test/hotspot/jtreg/compiler/valhalla/inlinetypes/TestIntrinsics.java +++ b/test/hotspot/jtreg/compiler/valhalla/inlinetypes/TestIntrinsics.java @@ -405,7 +405,7 @@ public void test21_verifier() { MyValue1 test22_vt; @Test - @IR(failOn = {CALL_UNSAFE, ALLOC}) + @IR(failOn = {CALL_UNSAFE}) public void test22(MyValue1 v) { v = U.makePrivateBuffer(v); U.putInt(v, X_OFFSET, rI);