[Wisp] Port Wisp1/2 to JDK17.

make/data/hotspot-symbols/symbols-unix

@@ -27,6 +27,7 @@ JVM_ArrayCopy
 JVM_AssertionStatusDirectives
 JVM_BeforeHalt
 JVM_CallStackWalk
+JVM_CheckAndClearNativeInterruptForWisp
 JVM_Clone
 JVM_ConstantPoolGetClassAt
 JVM_ConstantPoolGetClassAtIfLoaded
@@ -124,6 +125,7 @@ JVM_GetPermittedSubclasses
 JVM_GetPrimitiveArrayElement
 JVM_GetProperties
 JVM_GetProtectionDomain
+JVM_GetProxyUnpark
 JVM_GetRandomSeedForDumping
 JVM_GetRecordComponents
 JVM_GetSimpleBinaryName
@@ -148,6 +150,7 @@ JVM_IsCDSDumpingEnabled
 JVM_IsConstructorIx
 JVM_IsDumpingClassList
 JVM_IsHiddenClass
+JVM_IsInNative
 JVM_IsInterface
 JVM_IsPrimitiveClass
 JVM_IsRecord
@@ -161,6 +164,7 @@ JVM_LoadLibrary
 JVM_LookupDefineClass
 JVM_LookupLambdaProxyClassFromArchive
 JVM_LogLambdaFormInvoker
+JVM_MarkPreempt
 JVM_MaxMemory
 JVM_MaxObjectInspectionAge
 JVM_MonitorNotify
@@ -189,6 +193,7 @@ JVM_SetClassSigners
 JVM_SetNativeThreadName
 JVM_SetPrimitiveArrayElement
 JVM_SetThreadPriority
+JVM_SetWispTask
 JVM_Sleep
 JVM_StartThread
 JVM_StopThread

make/modules/java.base/Copy.gmk

@@ -196,6 +196,15 @@ $(NET_PROPERTIES_DST): $(NET_PROPERTIES_SRCS)
 
 TARGETS += $(NET_PROPERTIES_DST)
 
+################################################################################
+
+$(eval $(call SetupCopyFiles, COPY_WISP_PROPERTIES, \
+    FILES := $(TOPDIR)/src/java.base/share/conf/wisp.properties, \
+    DEST := $(CONF_DST_DIR), \
+))
+
+TARGETS += $(COPY_WISP_PROPERTIES)
+
 ifeq ($(call isTargetOs, linux), true)
   $(eval $(call SetupCopyFiles, COPY_SDP_CONF, \
       FILES := $(TOPDIR)/src/java.base/$(OPENJDK_TARGET_OS_TYPE)/conf/sdp/sdp.conf.template, \

src/hotspot/cpu/x86/c1_CodeStubs_x86.cpp

@@ -30,6 +30,7 @@
 #include "c1/c1_Runtime1.hpp"
 #include "classfile/javaClasses.hpp"
 #include "nativeInst_x86.hpp"
+#include "runtime/coroutine.hpp"
 #include "runtime/sharedRuntime.hpp"
 #include "utilities/align.hpp"
 #include "utilities/macros.hpp"
@@ -296,7 +297,27 @@ void MonitorExitStub::emit_code(LIR_Assembler* ce) {
   } else {
     exit_id = Runtime1::monitorexit_nofpu_id;
   }
+  // Handle spcecial case for wisp unpark.
+  // The code stub is entered only when the following four conditions are all satisfied
+  // 1. A synchronized method is compiled by C1
+  // 2. An exception happened in this method
+  // 3. There is no exception handler in this method, So it needs to unwind to its caller
+  // 4. GC happened during unpark
+  // if (_info == NULL) is true, the four condistions are all true.
+  if (UseWispMonitor && (_info == NULL)) {
+    if (exit_id == Runtime1::monitorexit_id) {
+      exit_id = Runtime1::monitorexit_proxy_id;
+    } else {
+      assert (exit_id == Runtime1::monitorexit_nofpu_id, "must be monitorexit_nofpu_id");
+      exit_id = Runtime1::monitorexit_nofpu_proxy_id;
+    }
+  }
   __ call(RuntimeAddress(Runtime1::entry_for(exit_id)));
+  // For direct unpark in Wisp, _info must be recorded to generate oopmap.
+  if (UseWispMonitor && _info) {
+    ce->add_call_info_here(_info);
+    ce->verify_oop_map(_info);
+  }
   __ jmp(_continuation);
 }
 
@@ -390,6 +411,10 @@ void PatchingStub::emit_code(LIR_Assembler* ce) {
     // begin_initialized_entry_offset has to fit in a byte. Also, we know it's not null.
     __ movptr(tmp2, Address(_obj, java_lang_Class::klass_offset()));
     __ get_thread(tmp);
+    if (UseWispMonitor) {
+      __ movptr(tmp, Address(tmp, JavaThread::current_coroutine_offset()));
+      __ movptr(tmp, Address(tmp, Coroutine::wisp_thread_offset()));
+    }
     __ cmpptr(tmp, Address(tmp2, InstanceKlass::init_thread_offset()));
     __ pop(tmp2);
     __ pop(tmp);

src/hotspot/cpu/x86/c1_LIRAssembler_x86.cpp

@@ -460,7 +460,7 @@ int LIR_Assembler::emit_unwind_handler() {
   MonitorExitStub* stub = NULL;
   if (method()->is_synchronized()) {
     monitor_address(0, FrameMap::rax_opr);
-    stub = new MonitorExitStub(FrameMap::rax_opr, true, 0);
+    stub = new MonitorExitStub(FrameMap::rax_opr, true, 0, NULL);
     __ unlock_object(rdi, rsi, rax, *stub->entry());
     __ bind(*stub->continuation());
   }

src/hotspot/cpu/x86/c1_LIRGenerator_x86.cpp

@@ -339,7 +339,16 @@ void LIRGenerator::do_MonitorExit(MonitorExit* x) {
   LIR_Opr lock = new_register(T_INT);
   LIR_Opr obj_temp = new_register(T_INT);
   set_no_result(x);
-  monitor_exit(obj_temp, lock, syncTempOpr(), LIR_OprFact::illegalOpr, x->monitor_no());
+  // Info is used to genereate oopmap, which is necessary for calling java code during rutime.
+  if (UseWispMonitor) {
+    CodeEmitInfo* info = NULL;
+    if (x->state()) {
+      info = state_for(x, x->state(), true);
+    }
+    monitor_exit(obj_temp, lock, syncTempOpr(), LIR_OprFact::illegalOpr, x->monitor_no(), NULL, info);
+  } else {
+    monitor_exit(obj_temp, lock, syncTempOpr(), LIR_OprFact::illegalOpr, x->monitor_no());
+  }
 }
 
 

src/hotspot/cpu/x86/c1_Runtime1_x86.cpp

@@ -40,12 +40,15 @@
 #include "oops/oop.inline.hpp"
 #include "prims/jvmtiExport.hpp"
 #include "register_x86.hpp"
+#include "runtime/coroutine.hpp"
 #include "runtime/sharedRuntime.hpp"
 #include "runtime/signature.hpp"
 #include "runtime/stubRoutines.hpp"
 #include "runtime/vframeArray.hpp"
 #include "utilities/macros.hpp"
 #include "vmreg_x86.inline.hpp"
+// the Runtime1::monitorenter function pointer
+extern address monitorenter_address_C1;
 
 // Implementation of StubAssembler
 
@@ -88,6 +91,14 @@ int StubAssembler::call_RT(Register oop_result1, Register metadata_result, addre
   if (!align_stack) {
     call_offset = offset();
   }
+
+  if (EnableSteal) {
+    // only if the entry_point is equal to `Runtime1::monitorenter()`, we will do this amendment.
+    if (entry == monitorenter_address_C1) {
+      WISP_V2v_UPDATE;
+    }
+  }
+
   // verify callee-saved register
 #ifdef ASSERT
   guarantee(thread != rax, "change this code");
@@ -1413,6 +1424,11 @@ OopMapSet* Runtime1::generate_code_for(StubID id, StubAssembler* sasm) {
         oop_maps = new OopMapSet();
         oop_maps->add_gc_map(call_offset, map);
         restore_live_registers(sasm, save_fpu_registers);
+
+        if (EnableSteal) {
+          // r15 has been forcely restored in restore_live_registers so we need fix it.
+          WISP_COMPILER_RESTORE_FORCE_UPDATE;
+        }
       }
       break;
 
@@ -1431,14 +1447,34 @@ OopMapSet* Runtime1::generate_code_for(StubID id, StubAssembler* sasm) {
         // note: really a leaf routine but must setup last java sp
         //       => use call_RT for now (speed can be improved by
         //       doing last java sp setup manually)
-        int call_offset = __ call_RT(noreg, noreg, CAST_FROM_FN_PTR(address, monitorexit), rax);
-
+        int call_offset;
+        if (UseWispMonitor) {
+          call_offset  = __ call_RT(noreg, noreg, CAST_FROM_FN_PTR(address, monitorexit_wisp), rax);
+        } else {
+          call_offset  = __ call_RT(noreg, noreg, CAST_FROM_FN_PTR(address, monitorexit), rax);
+        }
         oop_maps = new OopMapSet();
         oop_maps->add_gc_map(call_offset, map);
         restore_live_registers(sasm, save_fpu_registers);
       }
       break;
+    case monitorexit_nofpu_proxy_id:
+      save_fpu_registers = false;
+      // fall through
+    case monitorexit_proxy_id:
+      {
+        StubFrame f(sasm, "monitorexit", dont_gc_arguments);
+        OopMap* map = save_live_registers(sasm, 2, save_fpu_registers);
 
+        // Called with store_parameter and not C abi
+        f.load_argument(0, rax); // rax,: lock address
+        int call_offset;
+        call_offset  = __ call_RT(noreg, noreg, CAST_FROM_FN_PTR(address, monitorexit_wisp_proxy), rax);
+        oop_maps = new OopMapSet();
+        oop_maps->add_gc_map(call_offset, map);
+        restore_live_registers(sasm, save_fpu_registers);
+      }
+      break;
     case deoptimize_id:
       {
         StubFrame f(sasm, "deoptimize", dont_gc_arguments);

src/hotspot/cpu/x86/c2_MacroAssembler_x86.cpp

@@ -31,6 +31,7 @@
 #include "opto/opcodes.hpp"
 #include "opto/subnode.hpp"
 #include "runtime/biasedLocking.hpp"
+#include "runtime/coroutine.hpp"
 #include "runtime/objectMonitor.hpp"
 #include "runtime/stubRoutines.hpp"
 
@@ -597,15 +598,31 @@ void C2_MacroAssembler::fast_lock(Register objReg, Register boxReg, Register tmp
   // It's inflated and we use scrReg for ObjectMonitor* in this section.
   movq(scrReg, tmpReg);
   xorq(tmpReg, tmpReg);
+
+  // It's inflated and appears unlocked
+  if (UseWispMonitor) {
+    movptr (r15_thread, Address(r15_thread, JavaThread::current_coroutine_offset()));
+    movptr (r15_thread, Address(r15_thread, Coroutine::wisp_thread_offset()));
+  }
   lock();
   cmpxchgptr(r15_thread, Address(scrReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner)));
+  if (UseWispMonitor) {
+    movptr (r15_thread, Address(r15_thread, WispThread::thread_offset()));
+  }
   // Unconditionally set box->_displaced_header = markWord::unused_mark().
   // Without cast to int32_t this style of movptr will destroy r10 which is typically obj.
   movptr(Address(boxReg, 0), (int32_t)intptr_t(markWord::unused_mark().value()));
   // Propagate ICC.ZF from CAS above into DONE_LABEL.
   jcc(Assembler::equal, DONE_LABEL);           // CAS above succeeded; propagate ZF = 1 (success)
 
+  if (UseWispMonitor) {
+    movptr (r15_thread, Address(r15_thread, JavaThread::current_coroutine_offset()));
+    movptr (r15_thread, Address(r15_thread, Coroutine::wisp_thread_offset()));
+  }
   cmpptr(r15_thread, rax);                     // Check if we are already the owner (recursive lock)
+  if (UseWispMonitor) {
+    movptr (r15_thread, Address(r15_thread, WispThread::thread_offset()));
+  }
   jcc(Assembler::notEqual, DONE_LABEL);        // If not recursive, ZF = 0 at this point (fail)
   incq(Address(scrReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(recursions)));
   xorq(rax, rax); // Set ZF = 1 (success) for recursive lock, denoting locking success
@@ -688,7 +705,12 @@ void C2_MacroAssembler::fast_unlock(Register objReg, Register boxReg, Register t
   jcc   (Assembler::zero, DONE_LABEL);                              // 0 indicates recursive stack-lock
   movptr(tmpReg, Address(objReg, oopDesc::mark_offset_in_bytes())); // Examine the object's markword
   testptr(tmpReg, markWord::monitor_value);                         // Inflated?
-  jccb  (Assembler::zero, Stacked);
+  // If UseWispMonitor is enable, insert more code, then the length in jccb isn't enough.
+  if (UseWispMonitor) {
+    jcc  (Assembler::zero, Stacked);
+  } else {
+    jccb (Assembler::zero, Stacked);
+  }
 
   // It's inflated.
 #if INCLUDE_RTM_OPT
@@ -814,8 +836,15 @@ void C2_MacroAssembler::fast_unlock(Register objReg, Register boxReg, Register t
 
   // box is really RAX -- the following CMPXCHG depends on that binding
   // cmpxchg R,[M] is equivalent to rax = CAS(M,rax,R)
+  if (UseWispMonitor) {
+    movptr (r15_thread, Address(r15_thread, JavaThread::current_coroutine_offset()));
+    movptr (r15_thread, Address(r15_thread, Coroutine::wisp_thread_offset()));
+  }
   lock();
   cmpxchgptr(r15_thread, Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner)));
+  if (UseWispMonitor) {
+    movptr (r15_thread, Address(r15_thread, WispThread::thread_offset()));
+  }
   // There's no successor so we tried to regrab the lock.
   // If that didn't work, then another thread grabbed the
   // lock so we're done (and exit was a success).

src/hotspot/cpu/x86/interp_masm_x86.cpp

@@ -806,6 +806,7 @@ void InterpreterMacroAssembler::jump_from_interpreted(Register method, Register
 
   if (JvmtiExport::can_post_interpreter_events()) {
     Label run_compiled_code;
+    Label coroutine_skip_interpret;
     // JVMTI events, such as single-stepping, are implemented partly by avoiding running
     // compiled code in threads for which the event is enabled.  Check here for
     // interp_only_mode if these events CAN be enabled.
@@ -815,8 +816,19 @@ void InterpreterMacroAssembler::jump_from_interpreted(Register method, Register
     NOT_LP64(get_thread(temp);)
     cmpb(Address(temp, JavaThread::interp_only_mode_offset()), 0);
     jccb(Assembler::zero, run_compiled_code);
+    if (EnableCoroutine) {
+      cmpq(Address(method, Method::intrinsic_id_offset_in_bytes()), (int32_t)vmIntrinsics::_switchTo);
+      jcc(Assembler::zero, coroutine_skip_interpret);
+      cmpq(Address(method, Method::intrinsic_id_offset_in_bytes()), (int32_t)vmIntrinsics::_switchToAndExit);
+      jcc(Assembler::zero, coroutine_skip_interpret);
+      cmpq(Address(method, Method::intrinsic_id_offset_in_bytes()), (int32_t)vmIntrinsics::_switchToAndTerminate);
+      jcc(Assembler::zero, coroutine_skip_interpret);
+    }
     jmp(Address(method, Method::interpreter_entry_offset()));
     bind(run_compiled_code);
+    if (EnableCoroutine) {
+      bind(coroutine_skip_interpret);
+    }
   }
 
   jmp(Address(method, Method::from_interpreted_offset()));

src/hotspot/cpu/x86/macroAssembler_x86.cpp

@@ -41,6 +41,7 @@
 #include "oops/klass.inline.hpp"
 #include "prims/methodHandles.hpp"
 #include "runtime/biasedLocking.hpp"
+#include "runtime/coroutine.hpp"
 #include "runtime/flags/flagSetting.hpp"
 #include "runtime/interfaceSupport.inline.hpp"
 #include "runtime/jniHandles.hpp"
@@ -1278,6 +1279,11 @@ void MacroAssembler::bang_stack_size(Register size, Register tmp) {
 }
 
 void MacroAssembler::reserved_stack_check() {
+    //TODO: JDK11 introduces the usage of @ReservedStackAccess, we will support it with Coroutine
+    // If we check it in wisp flow, the exception is thrown when coroutine switches.
+    if (EnableCoroutine) {
+        return;
+    }
     // testing if reserved zone needs to be enabled
     Label no_reserved_zone_enabling;
     Register thread = NOT_LP64(rsi) LP64_ONLY(r15_thread);
@@ -1693,6 +1699,8 @@ void MacroAssembler::super_call_VM(Register oop_result,
   super_call_VM(oop_result, last_java_sp, entry_point, 3, check_exceptions);
 }
 
+extern address monitorenter_address_interp;
+
 void MacroAssembler::call_VM_base(Register oop_result,
                                   Register java_thread,
                                   Register last_java_sp,
@@ -1738,6 +1746,13 @@ void MacroAssembler::call_VM_base(Register oop_result,
   // do the call, remove parameters
   MacroAssembler::call_VM_leaf_base(entry_point, number_of_arguments);
 
+  if (EnableSteal) {
+    // only if the entry_point is equal to `Interpreter::monitorenter()`, we will do this amendment.
+    if (entry_point == monitorenter_address_interp) {
+      WISP_V2v_UPDATE;
+    }
+  }
+
   // restore the thread (cannot use the pushed argument since arguments
   // may be overwritten by C code generated by an optimizing compiler);
   // however can use the register value directly if it is callee saved.
@@ -1893,6 +1908,16 @@ void MacroAssembler::get_vm_result_2(Register metadata_result, Register java_thr
   movptr(Address(java_thread, JavaThread::vm_result_2_offset()), NULL_WORD);
 }
 
+void MacroAssembler::get_vm_result_for_wisp(Register oop_result, Register java_thread) {
+  assert(EnableCoroutine, "Coroutine is disabled");
+  movptr(oop_result, Address(java_thread, JavaThread::vm_result_for_wisp_offset()));
+  movptr(Address(java_thread, JavaThread::vm_result_for_wisp_offset()), NULL_WORD);
+  // In default flow, after calling get_vm_result, vm_result is set to null.
+  // In wisp flow, we should also clear vm_result.
+  movptr(Address(java_thread, JavaThread::vm_result_offset()), NULL_WORD);
+  verify_oop_msg(oop_result, "broken oop in call_VM_base");
+}
+
 void MacroAssembler::check_and_handle_earlyret(Register java_thread) {
 }
 
@@ -4136,8 +4161,15 @@ void MacroAssembler::clinit_barrier(Register klass, Register thread, Label* L_fa
   cmpb(Address(klass, InstanceKlass::init_state_offset()), InstanceKlass::fully_initialized);
   jcc(Assembler::equal, *L_fast_path);
 
+  if (UseWispMonitor) {
+    movptr(thread, Address(thread, JavaThread::current_coroutine_offset()));
+    movptr(thread, Address(thread, Coroutine::wisp_thread_offset()));
+  }
   // Fast path check: current thread is initializer thread
   cmpptr(thread, Address(klass, InstanceKlass::init_thread_offset()));
+  if (UseWispMonitor) {
+    movptr(thread, Address(thread, WispThread::thread_offset()));
+  }
   if (L_slow_path == &L_fallthrough) {
     jcc(Assembler::equal, *L_fast_path);
     bind(*L_slow_path);

src/hotspot/cpu/x86/macroAssembler_x86.hpp

@@ -274,6 +274,7 @@ class MacroAssembler: public Assembler {
 
   void get_vm_result  (Register oop_result, Register thread);
   void get_vm_result_2(Register metadata_result, Register thread);
+  void get_vm_result_for_wisp(Register oop_result, Register thread);
 
   // These always tightly bind to MacroAssembler::call_VM_base
   // bypassing the virtual implementation