From c5c5555bf4591f0195a2eddbc2de844d2a5a498d Mon Sep 17 00:00:00 2001 From: Zhai Mo Date: Thu, 29 Jun 2023 11:36:56 +0800 Subject: [PATCH 1/2] [Wisp] Port Wisp1/2 to JDK17. Summary: - Port Wisp1/2 to JDK11 includes the changes in 8.6.11_fp1 - initializing process improvement - WispCounterMXBean and WispPerfCounterMonitor aren't supported Test Plan: test/hotspot/jtreg/runtime/coroutine/ test/jdk/com/alibaba/wisp test/jdk/com/alibaba/wisp2 Reviewed-by: yulei Issue: https://github.com/dragonwell-project/dragonwell17/issues/77 --- make/data/hotspot-symbols/symbols-unix | 5 + make/modules/java.base/Copy.gmk | 7 + src/hotspot/cpu/x86/c1_CodeStubs_x86.cpp | 25 + src/hotspot/cpu/x86/c1_LIRAssembler_x86.cpp | 2 +- src/hotspot/cpu/x86/c1_LIRGenerator_x86.cpp | 11 +- src/hotspot/cpu/x86/c1_Runtime1_x86.cpp | 40 +- src/hotspot/cpu/x86/c2_MacroAssembler_x86.cpp | 24 +- src/hotspot/cpu/x86/interp_masm_x86.cpp | 12 + src/hotspot/cpu/x86/macroAssembler_x86.cpp | 25 + src/hotspot/cpu/x86/macroAssembler_x86.hpp | 1 + src/hotspot/cpu/x86/sharedRuntime_x86_64.cpp | 47 +- src/hotspot/cpu/x86/stubGenerator_x86_64.cpp | 14 + .../x86/templateInterpreterGenerator_x86.cpp | 32 +- src/hotspot/cpu/x86/x86.ad | 1 + src/hotspot/cpu/x86/x86_64.ad | 14 + src/hotspot/os/posix/os_posix.cpp | 14 + src/hotspot/share/c1/c1_CodeStubs.hpp | 16 +- src/hotspot/share/c1/c1_GraphBuilder.cpp | 24 +- src/hotspot/share/c1/c1_GraphBuilder.hpp | 1 + src/hotspot/share/c1/c1_Instruction.hpp | 6 + src/hotspot/share/c1/c1_LIR.cpp | 4 +- src/hotspot/share/c1/c1_LIR.hpp | 2 +- src/hotspot/share/c1/c1_LIRGenerator.cpp | 17 +- src/hotspot/share/c1/c1_LIRGenerator.hpp | 2 +- src/hotspot/share/c1/c1_Runtime1.cpp | 48 +- src/hotspot/share/c1/c1_Runtime1.hpp | 7 + src/hotspot/share/classfile/dictionary.cpp | 2 +- src/hotspot/share/classfile/javaClasses.cpp | 86 +- src/hotspot/share/classfile/javaClasses.hpp | 40 + src/hotspot/share/classfile/placeholders.cpp | 9 +- .../share/classfile/protectionDomainCache.cpp | 5 +- .../share/classfile/systemDictionary.cpp | 81 +- .../share/classfile/systemDictionary.hpp | 3 + .../classfile/systemDictionaryShared.cpp | 2 +- src/hotspot/share/classfile/vmClassMacros.hpp | 2 + src/hotspot/share/classfile/vmIntrinsics.hpp | 9 +- src/hotspot/share/classfile/vmSymbols.hpp | 28 + src/hotspot/share/include/jvm.h | 14 + .../share/interpreter/interpreterRuntime.cpp | 25 +- src/hotspot/share/jvmci/jvmciRuntime.cpp | 1 + src/hotspot/share/memory/allocation.hpp | 1 + src/hotspot/share/memory/arena.hpp | 3 + src/hotspot/share/memory/resourceArea.hpp | 3 + src/hotspot/share/oops/instanceKlass.cpp | 18 + src/hotspot/share/oops/instanceKlass.hpp | 4 +- src/hotspot/share/opto/classes.hpp | 1 + src/hotspot/share/opto/connode.hpp | 15 + src/hotspot/share/opto/generateOptoStub.cpp | 22 +- src/hotspot/share/opto/graphKit.cpp | 202 +++- src/hotspot/share/opto/graphKit.hpp | 5 +- src/hotspot/share/opto/machnode.cpp | 8 + src/hotspot/share/opto/macro.cpp | 18 +- src/hotspot/share/opto/matcher.cpp | 1 + src/hotspot/share/opto/parse1.cpp | 5 + src/hotspot/share/opto/runtime.cpp | 17 + src/hotspot/share/opto/runtime.hpp | 6 + src/hotspot/share/prims/jni.cpp | 28 +- src/hotspot/share/prims/jvm.cpp | 77 +- .../share/prims/jvmtiGetLoadedClasses.cpp | 2 +- src/hotspot/share/prims/jvmtiThreadState.cpp | 3 + src/hotspot/share/prims/unsafe.cpp | 58 +- src/hotspot/share/runtime/arguments.cpp | 49 + src/hotspot/share/runtime/coroutine.cpp | 615 +++++++++++- src/hotspot/share/runtime/coroutine.hpp | 291 +++++- src/hotspot/share/runtime/globals.hpp | 13 + src/hotspot/share/runtime/handles.hpp | 9 + src/hotspot/share/runtime/handles.inline.hpp | 11 + .../share/runtime/interfaceSupport.inline.hpp | 11 + src/hotspot/share/runtime/javaCalls.cpp | 12 +- src/hotspot/share/runtime/javaCalls.hpp | 4 + src/hotspot/share/runtime/mutexLocker.cpp | 49 +- src/hotspot/share/runtime/mutexLocker.hpp | 38 +- src/hotspot/share/runtime/objectMonitor.cpp | 119 ++- src/hotspot/share/runtime/objectMonitor.hpp | 5 + .../share/runtime/objectMonitor.inline.hpp | 4 + src/hotspot/share/runtime/os.cpp | 4 + src/hotspot/share/runtime/os.hpp | 4 + src/hotspot/share/runtime/reflection.cpp | 17 +- src/hotspot/share/runtime/sharedRuntime.cpp | 74 +- src/hotspot/share/runtime/sharedRuntime.hpp | 2 + src/hotspot/share/runtime/synchronizer.cpp | 122 ++- src/hotspot/share/runtime/synchronizer.hpp | 35 + src/hotspot/share/runtime/thread.cpp | 101 +- src/hotspot/share/runtime/thread.hpp | 19 +- src/hotspot/share/runtime/thread.inline.hpp | 2 +- src/hotspot/share/runtime/vframe.cpp | 2 +- src/hotspot/share/runtime/vframe.hpp | 4 + src/hotspot/share/services/threadService.hpp | 5 + src/hotspot/share/utilities/exceptions.cpp | 14 + .../linux/classes/sun/nio/ch/EPoll.java | 48 + .../classes/sun/nio/ch/EPollSelectorImpl.java | 24 +- .../sun/nio/ch/EPollSelectorProvider.java | 13 + src/java.base/linux/native/libnio/ch/EPoll.c | 5 + .../wisp/engine/ScheduledWispEngine.java | 621 ++++++++++++ .../wisp/engine/StealAwareRunnable.java | 21 + .../com/alibaba/wisp/engine/TimeOut.java | 221 ++++ .../com/alibaba/wisp/engine/Wisp2Engine.java | 354 +++++++ .../com/alibaba/wisp/engine/Wisp2Group.java | 128 +++ .../alibaba/wisp/engine/Wisp2Scheduler.java | 350 +++++++ .../wisp/engine/WispConfiguration.java | 247 +++++ .../com/alibaba/wisp/engine/WispCounter.java | 222 ++++ .../com/alibaba/wisp/engine/WispEngine.java | 944 ++++++++++++++++++ .../com/alibaba/wisp/engine/WispPoller.java | 127 +++ .../com/alibaba/wisp/engine/WispSysmon.java | 88 ++ .../com/alibaba/wisp/engine/WispTask.java | 584 +++++++++++ .../wisp/engine/WispThreadWrapper.java | 82 ++ .../wisp/engine/WispWorkerContainer.java | 113 +++ .../alibaba/wisp/util/io/SleepSelector.java | 158 +++ .../alibaba/wisp/util/io/WispInputStream.java | 128 +++ .../wisp/util/io/WispOutputStream.java | 95 ++ .../share/classes/java/dyn/Coroutine.java | 143 ++- .../share/classes/java/dyn/CoroutineBase.java | 116 ++- .../java/dyn/CoroutineExitException.java | 29 +- .../classes/java/dyn/CoroutineSupport.java | 531 ++++++---- .../share/classes/java/lang/System.java | 52 + .../share/classes/java/lang/Thread.java | 179 +++- .../classes/java/net/DatagramSocket.java | 103 +- .../share/classes/java/net/ServerSocket.java | 89 +- .../share/classes/java/net/Socket.java | 204 +++- .../util/concurrent/LinkedBlockingDeque.java | 3 + .../util/concurrent/LinkedBlockingQueue.java | 4 + .../ScheduledThreadPoolExecutor.java | 8 + .../util/concurrent/SynchronousQueue.java | 12 +- .../util/concurrent/ThreadPoolExecutor.java | 136 +++ .../locks/AbstractQueuedSynchronizer.java | 9 +- .../jdk/internal/access/JavaLangAccess.java | 25 + .../jdk/internal/access/SharedSecrets.java | 29 + .../jdk/internal/access/UnsafeAccess.java | 7 + .../jdk/internal/access/WispEngineAccess.java | 76 ++ .../classes/jdk/internal/misc/Unsafe.java | 53 +- src/java.base/share/classes/module-info.java | 1 + .../share/classes/sun/nio/ch/EpollAccess.java | 35 + .../sun/nio/ch/ServerSocketChannelImpl.java | 17 +- .../sun/nio/ch/WispServerSocketImpl.java | 166 +++ .../classes/sun/nio/ch/WispSocketImpl.java | 378 +++++++ .../classes/sun/nio/ch/WispUdpSocketImpl.java | 348 +++++++ src/java.base/share/conf/wisp.properties | 19 + .../share/native/libjava/Coroutine.c | 18 + src/java.base/share/native/libjava/Thread.c | 9 +- .../share/native/libjava/WispEngine.c | 15 + .../share/native/libjava/WispSysmon.c | 15 + src/java.base/share/native/libjava/WispTask.c | 17 + .../coroutine/C1ThrowSyncExceptionTest.java | 85 ++ .../CoroutineNativeConcurrentTest.java | 136 +++ .../runtime/coroutine/DirectUnparkTest.java | 55 + .../coroutine/InterruptedWaitTest.java | 34 + .../jtreg/runtime/coroutine/JStackTest.java | 133 +++ .../coroutine/MultiCoroutineStackTest.java | 133 +++ .../coroutine/PremainWithWispMonitorTest.java | 41 + .../runtime/coroutine/SimpleWispTest.java | 13 + .../TestAvoidDeoptCoroutineMethod.java | 33 + .../runtime/coroutine/TimeSliceSyncTest.java | 121 +++ .../runtime/coroutine/Wisp2SwitchTest.java | 58 ++ .../runtime/coroutine/Wisp2SwitchTest2.java | 59 ++ .../runtime/coroutine/WispClinitTest.java | 35 + .../coroutine/WispEmitNewGuardTest.java | 32 + .../runtime/coroutine/c1AssertFailTest.sh | 18 + .../jniDetachThreadHoldingMonitorTest.c | 76 ++ .../jniDetachThreadHoldingMonitorTest.sh | 18 + .../runtime/coroutine/jniMonitorExitTest.c | 53 + .../runtime/coroutine/jniMonitorExitTest.sh | 19 + .../runtime/coroutine/jvmtiWispMonitorTest.sh | 19 + .../runtime/coroutine/logCompilationTest.sh | 33 + .../coroutine/premainWithWispMonitorTest.sh | 36 + .../ConfigurationCompatibilityCheckTest.java | 33 + test/jdk/com/alibaba/wisp/CoroutineTest.java | 55 + test/jdk/com/alibaba/wisp/ExecutionTest.java | 124 +++ test/jdk/com/alibaba/wisp/IoTest.java | 85 ++ test/jdk/com/alibaba/wisp/ParkTest.java | 65 ++ .../alibaba/wisp/SelectorLazyCreateTest.java | 32 + .../wisp/boot/UnsafeDependencyBugTest.java | 37 + .../wisp/bug/CancelTimerAndSleepTest.java | 28 + .../com/alibaba/wisp/bug/ClearEventTest.java | 84 ++ .../alibaba/wisp/bug/Id2TaskMapLeakTest.java | 37 + .../bug/ReleaseWispSocketAndExitTest.java | 88 ++ .../wisp/bug/ResetTaskCancelTimerBugTest.java | 49 + .../wisp/bug/SelectorInitCriticalTest.java | 32 + .../alibaba/wisp/bug/TestThreadStackTrace.sh | 134 +++ .../com/alibaba/wisp/bug/ThreadLockTest.java | 75 ++ .../bug/ThreadPoolFastShutdownBugTest.java | 26 + .../bug/WispEngineCriticalSectionTest.java | 88 ++ .../wisp/bug/WispSelectorReadyOpsTest.java | 45 + .../WispSocketLeakWhenConnectTimeoutTest.java | 28 + .../alibaba/wisp/close/WispDestroyTest.java | 38 + .../wisp/env/CtxClassLoaderIsolateTest.java | 26 + .../alibaba/wisp/io/CreateFdOnDemandTest.java | 61 ++ .../alibaba/wisp/io/DatagramSocketTest.java | 87 ++ .../com/alibaba/wisp/io/GlobalPollerTest.java | 53 + .../wisp/io/ReuseUdpSocektBufTest.java | 72 ++ .../alibaba/wisp/io/SelectorRebuildTest.java | 83 ++ test/jdk/com/alibaba/wisp/io/SocketTest.java | 96 ++ test/jdk/com/alibaba/wisp/lock/AQSTest.java | 51 + .../alibaba/wisp/lock/ElisionSpinTest.java | 42 + test/jdk/com/alibaba/wisp/lock/LockTest.java | 72 ++ .../wisp/lock/LockUninterruptiblyTest.java | 57 ++ .../com/alibaba/wisp/lock/ParkNanoTest.java | 27 + .../com/alibaba/wisp/lock/UnsafeParkTest.java | 44 + .../wisp/monitor/C2SyncMethodTest.java | 19 + .../alibaba/wisp/monitor/FinalizerTest.java | 35 + .../alibaba/wisp/monitor/JNICriticalTest.java | 58 ++ .../wisp/monitor/LazyUnparkBugTest.java | 55 + .../alibaba/wisp/monitor/MultiThreadTest.java | 146 +++ .../wisp/monitor/SynchronizedTest.java | 28 + .../alibaba/wisp/monitor/WaitNotifyTest.java | 65 ++ .../alibaba/wisp/monitor/WispExitTest.java | 11 + .../DisableThreadAsWispAtRuntimeTest.java | 79 ++ .../wisp/thread/EngineExecutorTest.java | 47 + .../alibaba/wisp/thread/InterruptTest.java | 78 ++ .../wisp/thread/InterruptedSleepTest.java | 36 + .../com/alibaba/wisp/thread/IsAliveTest.java | 58 ++ .../wisp/thread/SubmittedTaskLimitTest.java | 62 ++ .../alibaba/wisp/thread/ThreadAsWispTest.java | 227 +++++ .../alibaba/wisp/thread/ThrowErrorTest.java | 28 + .../alibaba/wisp/thread/TimeSliceTest.java | 45 + .../com/alibaba/wisp/thread/YieldTest.java | 29 + .../wisp/timer/DaemonThreadGroupTest.java | 63 ++ .../com/alibaba/wisp/timer/OverflowTest.java | 49 + .../wisp/timer/PriorityQueueSortTest.java | 131 +++ .../com/alibaba/wisp/timer/SleepRPCTest.java | 50 + .../jdk/com/alibaba/wisp/timer/SleepTest.java | 42 + .../jdk/com/alibaba/wisp/timer/TimerTest.java | 34 + .../alibaba/wisp2/AllThreadAsWispTest.java | 50 + .../wisp2/CtxClassLoaderInheritanceTest.java | 29 + test/jdk/com/alibaba/wisp2/DispatchTest.java | 27 + test/jdk/com/alibaba/wisp2/HandOffTest.java | 50 + .../alibaba/wisp2/NioBlockingAcceptTest.java | 34 + .../ReuseWispTaskAfterThreadJoinTest.java | 30 + .../jdk/com/alibaba/wisp2/ThreadJoinTest.java | 37 + test/jdk/com/alibaba/wisp2/TimedWaitTest.java | 30 + .../com/alibaba/wisp2/Wisp2ShutdownTest.java | 66 ++ .../alibaba/wisp2/Wisp2WaitNotifyTest.java | 70 ++ .../com/alibaba/wisp2/Wisp2WorkStealTest.java | 63 ++ .../jdk/com/alibaba/wisp2/Wisp2YieldTest.java | 41 + .../wisp2/bug/ConcurrentThreadJoinTest.java | 22 + .../wisp2/bug/DisableStealBugTest.java | 57 ++ .../Wisp2ThreadObjLeakInThreadGroupTest.java | 32 + test/jdk/java/dyn/BasicStealTest.java | 57 ++ test/jdk/java/dyn/ConcurrentStealTest.java | 106 ++ 238 files changed, 15183 insertions(+), 546 deletions(-) create mode 100644 src/java.base/share/classes/com/alibaba/wisp/engine/ScheduledWispEngine.java create mode 100644 src/java.base/share/classes/com/alibaba/wisp/engine/StealAwareRunnable.java create mode 100644 src/java.base/share/classes/com/alibaba/wisp/engine/TimeOut.java create mode 100644 src/java.base/share/classes/com/alibaba/wisp/engine/Wisp2Engine.java create mode 100644 src/java.base/share/classes/com/alibaba/wisp/engine/Wisp2Group.java create mode 100644 src/java.base/share/classes/com/alibaba/wisp/engine/Wisp2Scheduler.java create mode 100644 src/java.base/share/classes/com/alibaba/wisp/engine/WispConfiguration.java create mode 100644 src/java.base/share/classes/com/alibaba/wisp/engine/WispCounter.java create mode 100644 src/java.base/share/classes/com/alibaba/wisp/engine/WispEngine.java create mode 100644 src/java.base/share/classes/com/alibaba/wisp/engine/WispPoller.java create mode 100644 src/java.base/share/classes/com/alibaba/wisp/engine/WispSysmon.java create mode 100644 src/java.base/share/classes/com/alibaba/wisp/engine/WispTask.java create mode 100644 src/java.base/share/classes/com/alibaba/wisp/engine/WispThreadWrapper.java create mode 100644 src/java.base/share/classes/com/alibaba/wisp/engine/WispWorkerContainer.java create mode 100644 src/java.base/share/classes/com/alibaba/wisp/util/io/SleepSelector.java create mode 100644 src/java.base/share/classes/com/alibaba/wisp/util/io/WispInputStream.java create mode 100644 src/java.base/share/classes/com/alibaba/wisp/util/io/WispOutputStream.java create mode 100644 src/java.base/share/classes/jdk/internal/access/UnsafeAccess.java create mode 100644 src/java.base/share/classes/jdk/internal/access/WispEngineAccess.java create mode 100644 src/java.base/share/classes/sun/nio/ch/EpollAccess.java create mode 100644 src/java.base/share/classes/sun/nio/ch/WispServerSocketImpl.java create mode 100644 src/java.base/share/classes/sun/nio/ch/WispSocketImpl.java create mode 100644 src/java.base/share/classes/sun/nio/ch/WispUdpSocketImpl.java create mode 100644 src/java.base/share/conf/wisp.properties create mode 100644 src/java.base/share/native/libjava/Coroutine.c create mode 100644 src/java.base/share/native/libjava/WispEngine.c create mode 100644 src/java.base/share/native/libjava/WispSysmon.c create mode 100644 src/java.base/share/native/libjava/WispTask.c create mode 100644 test/hotspot/jtreg/runtime/coroutine/C1ThrowSyncExceptionTest.java create mode 100644 test/hotspot/jtreg/runtime/coroutine/CoroutineNativeConcurrentTest.java create mode 100644 test/hotspot/jtreg/runtime/coroutine/DirectUnparkTest.java create mode 100644 test/hotspot/jtreg/runtime/coroutine/InterruptedWaitTest.java create mode 100644 test/hotspot/jtreg/runtime/coroutine/JStackTest.java create mode 100644 test/hotspot/jtreg/runtime/coroutine/MultiCoroutineStackTest.java create mode 100644 test/hotspot/jtreg/runtime/coroutine/PremainWithWispMonitorTest.java create mode 100644 test/hotspot/jtreg/runtime/coroutine/SimpleWispTest.java create mode 100644 test/hotspot/jtreg/runtime/coroutine/TestAvoidDeoptCoroutineMethod.java create mode 100644 test/hotspot/jtreg/runtime/coroutine/TimeSliceSyncTest.java create mode 100644 test/hotspot/jtreg/runtime/coroutine/Wisp2SwitchTest.java create mode 100644 test/hotspot/jtreg/runtime/coroutine/Wisp2SwitchTest2.java create mode 100644 test/hotspot/jtreg/runtime/coroutine/WispClinitTest.java create mode 100644 test/hotspot/jtreg/runtime/coroutine/WispEmitNewGuardTest.java create mode 100644 test/hotspot/jtreg/runtime/coroutine/c1AssertFailTest.sh create mode 100644 test/hotspot/jtreg/runtime/coroutine/jniDetachThreadHoldingMonitorTest.c create mode 100644 test/hotspot/jtreg/runtime/coroutine/jniDetachThreadHoldingMonitorTest.sh create mode 100644 test/hotspot/jtreg/runtime/coroutine/jniMonitorExitTest.c create mode 100644 test/hotspot/jtreg/runtime/coroutine/jniMonitorExitTest.sh create mode 100644 test/hotspot/jtreg/runtime/coroutine/jvmtiWispMonitorTest.sh create mode 100644 test/hotspot/jtreg/runtime/coroutine/logCompilationTest.sh create mode 100644 test/hotspot/jtreg/runtime/coroutine/premainWithWispMonitorTest.sh create mode 100644 test/jdk/com/alibaba/wisp/ConfigurationCompatibilityCheckTest.java create mode 100644 test/jdk/com/alibaba/wisp/CoroutineTest.java create mode 100644 test/jdk/com/alibaba/wisp/ExecutionTest.java create mode 100644 test/jdk/com/alibaba/wisp/IoTest.java create mode 100644 test/jdk/com/alibaba/wisp/ParkTest.java create mode 100644 test/jdk/com/alibaba/wisp/SelectorLazyCreateTest.java create mode 100644 test/jdk/com/alibaba/wisp/boot/UnsafeDependencyBugTest.java create mode 100644 test/jdk/com/alibaba/wisp/bug/CancelTimerAndSleepTest.java create mode 100644 test/jdk/com/alibaba/wisp/bug/ClearEventTest.java create mode 100644 test/jdk/com/alibaba/wisp/bug/Id2TaskMapLeakTest.java create mode 100644 test/jdk/com/alibaba/wisp/bug/ReleaseWispSocketAndExitTest.java create mode 100644 test/jdk/com/alibaba/wisp/bug/ResetTaskCancelTimerBugTest.java create mode 100644 test/jdk/com/alibaba/wisp/bug/SelectorInitCriticalTest.java create mode 100644 test/jdk/com/alibaba/wisp/bug/TestThreadStackTrace.sh create mode 100644 test/jdk/com/alibaba/wisp/bug/ThreadLockTest.java create mode 100644 test/jdk/com/alibaba/wisp/bug/ThreadPoolFastShutdownBugTest.java create mode 100644 test/jdk/com/alibaba/wisp/bug/WispEngineCriticalSectionTest.java create mode 100644 test/jdk/com/alibaba/wisp/bug/WispSelectorReadyOpsTest.java create mode 100644 test/jdk/com/alibaba/wisp/bug/WispSocketLeakWhenConnectTimeoutTest.java create mode 100644 test/jdk/com/alibaba/wisp/close/WispDestroyTest.java create mode 100644 test/jdk/com/alibaba/wisp/env/CtxClassLoaderIsolateTest.java create mode 100644 test/jdk/com/alibaba/wisp/io/CreateFdOnDemandTest.java create mode 100644 test/jdk/com/alibaba/wisp/io/DatagramSocketTest.java create mode 100644 test/jdk/com/alibaba/wisp/io/GlobalPollerTest.java create mode 100644 test/jdk/com/alibaba/wisp/io/ReuseUdpSocektBufTest.java create mode 100644 test/jdk/com/alibaba/wisp/io/SelectorRebuildTest.java create mode 100644 test/jdk/com/alibaba/wisp/io/SocketTest.java create mode 100644 test/jdk/com/alibaba/wisp/lock/AQSTest.java create mode 100644 test/jdk/com/alibaba/wisp/lock/ElisionSpinTest.java create mode 100644 test/jdk/com/alibaba/wisp/lock/LockTest.java create mode 100644 test/jdk/com/alibaba/wisp/lock/LockUninterruptiblyTest.java create mode 100644 test/jdk/com/alibaba/wisp/lock/ParkNanoTest.java create mode 100644 test/jdk/com/alibaba/wisp/lock/UnsafeParkTest.java create mode 100644 test/jdk/com/alibaba/wisp/monitor/C2SyncMethodTest.java create mode 100644 test/jdk/com/alibaba/wisp/monitor/FinalizerTest.java create mode 100644 test/jdk/com/alibaba/wisp/monitor/JNICriticalTest.java create mode 100644 test/jdk/com/alibaba/wisp/monitor/LazyUnparkBugTest.java create mode 100644 test/jdk/com/alibaba/wisp/monitor/MultiThreadTest.java create mode 100644 test/jdk/com/alibaba/wisp/monitor/SynchronizedTest.java create mode 100644 test/jdk/com/alibaba/wisp/monitor/WaitNotifyTest.java create mode 100644 test/jdk/com/alibaba/wisp/monitor/WispExitTest.java create mode 100644 test/jdk/com/alibaba/wisp/thread/DisableThreadAsWispAtRuntimeTest.java create mode 100644 test/jdk/com/alibaba/wisp/thread/EngineExecutorTest.java create mode 100644 test/jdk/com/alibaba/wisp/thread/InterruptTest.java create mode 100644 test/jdk/com/alibaba/wisp/thread/InterruptedSleepTest.java create mode 100644 test/jdk/com/alibaba/wisp/thread/IsAliveTest.java create mode 100644 test/jdk/com/alibaba/wisp/thread/SubmittedTaskLimitTest.java create mode 100644 test/jdk/com/alibaba/wisp/thread/ThreadAsWispTest.java create mode 100644 test/jdk/com/alibaba/wisp/thread/ThrowErrorTest.java create mode 100644 test/jdk/com/alibaba/wisp/thread/TimeSliceTest.java create mode 100644 test/jdk/com/alibaba/wisp/thread/YieldTest.java create mode 100644 test/jdk/com/alibaba/wisp/timer/DaemonThreadGroupTest.java create mode 100644 test/jdk/com/alibaba/wisp/timer/OverflowTest.java create mode 100644 test/jdk/com/alibaba/wisp/timer/PriorityQueueSortTest.java create mode 100644 test/jdk/com/alibaba/wisp/timer/SleepRPCTest.java create mode 100644 test/jdk/com/alibaba/wisp/timer/SleepTest.java create mode 100644 test/jdk/com/alibaba/wisp/timer/TimerTest.java create mode 100644 test/jdk/com/alibaba/wisp2/AllThreadAsWispTest.java create mode 100644 test/jdk/com/alibaba/wisp2/CtxClassLoaderInheritanceTest.java create mode 100644 test/jdk/com/alibaba/wisp2/DispatchTest.java create mode 100644 test/jdk/com/alibaba/wisp2/HandOffTest.java create mode 100644 test/jdk/com/alibaba/wisp2/NioBlockingAcceptTest.java create mode 100644 test/jdk/com/alibaba/wisp2/ReuseWispTaskAfterThreadJoinTest.java create mode 100644 test/jdk/com/alibaba/wisp2/ThreadJoinTest.java create mode 100644 test/jdk/com/alibaba/wisp2/TimedWaitTest.java create mode 100644 test/jdk/com/alibaba/wisp2/Wisp2ShutdownTest.java create mode 100644 test/jdk/com/alibaba/wisp2/Wisp2WaitNotifyTest.java create mode 100644 test/jdk/com/alibaba/wisp2/Wisp2WorkStealTest.java create mode 100644 test/jdk/com/alibaba/wisp2/Wisp2YieldTest.java create mode 100644 test/jdk/com/alibaba/wisp2/bug/ConcurrentThreadJoinTest.java create mode 100644 test/jdk/com/alibaba/wisp2/bug/DisableStealBugTest.java create mode 100644 test/jdk/com/alibaba/wisp2/bug/Wisp2ThreadObjLeakInThreadGroupTest.java create mode 100644 test/jdk/java/dyn/BasicStealTest.java create mode 100644 test/jdk/java/dyn/ConcurrentStealTest.java diff --git a/make/data/hotspot-symbols/symbols-unix b/make/data/hotspot-symbols/symbols-unix index a34732739d3..494b924a839 100644 --- a/make/data/hotspot-symbols/symbols-unix +++ b/make/data/hotspot-symbols/symbols-unix @@ -27,6 +27,7 @@ JVM_ArrayCopy JVM_AssertionStatusDirectives JVM_BeforeHalt JVM_CallStackWalk +JVM_CheckAndClearNativeInterruptForWisp JVM_Clone JVM_ConstantPoolGetClassAt JVM_ConstantPoolGetClassAtIfLoaded @@ -124,6 +125,7 @@ JVM_GetPermittedSubclasses JVM_GetPrimitiveArrayElement JVM_GetProperties JVM_GetProtectionDomain +JVM_GetProxyUnpark JVM_GetRandomSeedForDumping JVM_GetRecordComponents JVM_GetSimpleBinaryName @@ -148,6 +150,7 @@ JVM_IsCDSDumpingEnabled JVM_IsConstructorIx JVM_IsDumpingClassList JVM_IsHiddenClass +JVM_IsInNative JVM_IsInterface JVM_IsPrimitiveClass JVM_IsRecord @@ -161,6 +164,7 @@ JVM_LoadLibrary JVM_LookupDefineClass JVM_LookupLambdaProxyClassFromArchive JVM_LogLambdaFormInvoker +JVM_MarkPreempt JVM_MaxMemory JVM_MaxObjectInspectionAge JVM_MonitorNotify @@ -189,6 +193,7 @@ JVM_SetClassSigners JVM_SetNativeThreadName JVM_SetPrimitiveArrayElement JVM_SetThreadPriority +JVM_SetWispTask JVM_Sleep JVM_StartThread JVM_StopThread diff --git a/make/modules/java.base/Copy.gmk b/make/modules/java.base/Copy.gmk index dfa2a54f63a..b2ceb409d6a 100644 --- a/make/modules/java.base/Copy.gmk +++ b/make/modules/java.base/Copy.gmk @@ -205,6 +205,13 @@ ifeq ($(call isTargetOs, linux), true) TARGETS += $(COPY_SDP_CONF) endif +$(eval $(call SetupCopyFiles, COPY_WISP_PROPERTIES, \ + FILES := $(TOPDIR)/src/java.base/share/conf/wisp.properties, \ + DEST := $(CONF_DST_DIR), \ +)) + +TARGETS += $(COPY_WISP_PROPERTIES) + ################################################################################ # JDK license and assembly exception files to be packaged in JMOD diff --git a/src/hotspot/cpu/x86/c1_CodeStubs_x86.cpp b/src/hotspot/cpu/x86/c1_CodeStubs_x86.cpp index 1ba5061a574..8a8d2a8ccb2 100644 --- a/src/hotspot/cpu/x86/c1_CodeStubs_x86.cpp +++ b/src/hotspot/cpu/x86/c1_CodeStubs_x86.cpp @@ -30,6 +30,7 @@ #include "c1/c1_Runtime1.hpp" #include "classfile/javaClasses.hpp" #include "nativeInst_x86.hpp" +#include "runtime/coroutine.hpp" #include "runtime/sharedRuntime.hpp" #include "utilities/align.hpp" #include "utilities/macros.hpp" @@ -296,7 +297,27 @@ void MonitorExitStub::emit_code(LIR_Assembler* ce) { } else { exit_id = Runtime1::monitorexit_nofpu_id; } + // Handle spcecial case for wisp unpark. + // The code stub is entered only when the following four conditions are all satisfied + // 1. A synchronized method is compiled by C1 + // 2. An exception happened in this method + // 3. There is no exception handler in this method, So it needs to unwind to its caller + // 4. GC happened during unpark + // if (_info == NULL) is true, the four condistions are all true. + if (UseWispMonitor && (_info == NULL)) { + if (exit_id == Runtime1::monitorexit_id) { + exit_id = Runtime1::monitorexit_proxy_id; + } else { + assert (exit_id == Runtime1::monitorexit_nofpu_id, "must be monitorexit_nofpu_id"); + exit_id = Runtime1::monitorexit_nofpu_proxy_id; + } + } __ call(RuntimeAddress(Runtime1::entry_for(exit_id))); + // For direct unpark in Wisp, _info must be recorded to generate oopmap. + if (UseWispMonitor && _info) { + ce->add_call_info_here(_info); + ce->verify_oop_map(_info); + } __ jmp(_continuation); } @@ -390,6 +411,10 @@ void PatchingStub::emit_code(LIR_Assembler* ce) { // begin_initialized_entry_offset has to fit in a byte. Also, we know it's not null. __ movptr(tmp2, Address(_obj, java_lang_Class::klass_offset())); __ get_thread(tmp); + if (UseWispMonitor) { + __ movptr(tmp, Address(tmp, JavaThread::current_coroutine_offset())); + __ movptr(tmp, Address(tmp, Coroutine::wisp_thread_offset())); + } __ cmpptr(tmp, Address(tmp2, InstanceKlass::init_thread_offset())); __ pop(tmp2); __ pop(tmp); diff --git a/src/hotspot/cpu/x86/c1_LIRAssembler_x86.cpp b/src/hotspot/cpu/x86/c1_LIRAssembler_x86.cpp index e2454f32481..d9628a5fb13 100644 --- a/src/hotspot/cpu/x86/c1_LIRAssembler_x86.cpp +++ b/src/hotspot/cpu/x86/c1_LIRAssembler_x86.cpp @@ -460,7 +460,7 @@ int LIR_Assembler::emit_unwind_handler() { MonitorExitStub* stub = NULL; if (method()->is_synchronized()) { monitor_address(0, FrameMap::rax_opr); - stub = new MonitorExitStub(FrameMap::rax_opr, true, 0); + stub = new MonitorExitStub(FrameMap::rax_opr, true, 0, NULL); __ unlock_object(rdi, rsi, rax, *stub->entry()); __ bind(*stub->continuation()); } diff --git a/src/hotspot/cpu/x86/c1_LIRGenerator_x86.cpp b/src/hotspot/cpu/x86/c1_LIRGenerator_x86.cpp index b99f16fea05..17c5dcc7268 100644 --- a/src/hotspot/cpu/x86/c1_LIRGenerator_x86.cpp +++ b/src/hotspot/cpu/x86/c1_LIRGenerator_x86.cpp @@ -339,7 +339,16 @@ void LIRGenerator::do_MonitorExit(MonitorExit* x) { LIR_Opr lock = new_register(T_INT); LIR_Opr obj_temp = new_register(T_INT); set_no_result(x); - monitor_exit(obj_temp, lock, syncTempOpr(), LIR_OprFact::illegalOpr, x->monitor_no()); + // Info is used to genereate oopmap, which is necessary for calling java code during rutime. + if (UseWispMonitor) { + CodeEmitInfo* info = NULL; + if (x->state()) { + info = state_for(x, x->state(), true); + } + monitor_exit(obj_temp, lock, syncTempOpr(), LIR_OprFact::illegalOpr, x->monitor_no(), NULL, info); + } else { + monitor_exit(obj_temp, lock, syncTempOpr(), LIR_OprFact::illegalOpr, x->monitor_no()); + } } diff --git a/src/hotspot/cpu/x86/c1_Runtime1_x86.cpp b/src/hotspot/cpu/x86/c1_Runtime1_x86.cpp index 42c46496d47..0ada550a044 100644 --- a/src/hotspot/cpu/x86/c1_Runtime1_x86.cpp +++ b/src/hotspot/cpu/x86/c1_Runtime1_x86.cpp @@ -40,12 +40,15 @@ #include "oops/oop.inline.hpp" #include "prims/jvmtiExport.hpp" #include "register_x86.hpp" +#include "runtime/coroutine.hpp" #include "runtime/sharedRuntime.hpp" #include "runtime/signature.hpp" #include "runtime/stubRoutines.hpp" #include "runtime/vframeArray.hpp" #include "utilities/macros.hpp" #include "vmreg_x86.inline.hpp" +// the Runtime1::monitorenter function pointer +extern address monitorenter_address_C1; // Implementation of StubAssembler @@ -88,6 +91,14 @@ int StubAssembler::call_RT(Register oop_result1, Register metadata_result, addre if (!align_stack) { call_offset = offset(); } + + if (EnableSteal) { + // only if the entry_point is equal to `Runtime1::monitorenter()`, we will do this amendment. + if (entry == monitorenter_address_C1) { + WISP_V2v_UPDATE; + } + } + // verify callee-saved register #ifdef ASSERT guarantee(thread != rax, "change this code"); @@ -1413,6 +1424,11 @@ OopMapSet* Runtime1::generate_code_for(StubID id, StubAssembler* sasm) { oop_maps = new OopMapSet(); oop_maps->add_gc_map(call_offset, map); restore_live_registers(sasm, save_fpu_registers); + + if (EnableSteal) { + // r15 has been forcely restored in restore_live_registers so we need fix it. + WISP_COMPILER_RESTORE_FORCE_UPDATE; + } } break; @@ -1431,14 +1447,34 @@ OopMapSet* Runtime1::generate_code_for(StubID id, StubAssembler* sasm) { // note: really a leaf routine but must setup last java sp // => use call_RT for now (speed can be improved by // doing last java sp setup manually) - int call_offset = __ call_RT(noreg, noreg, CAST_FROM_FN_PTR(address, monitorexit), rax); - + int call_offset; + if (UseWispMonitor) { + call_offset = __ call_RT(noreg, noreg, CAST_FROM_FN_PTR(address, monitorexit_wisp), rax); + } else { + call_offset = __ call_RT(noreg, noreg, CAST_FROM_FN_PTR(address, monitorexit), rax); + } oop_maps = new OopMapSet(); oop_maps->add_gc_map(call_offset, map); restore_live_registers(sasm, save_fpu_registers); } break; + case monitorexit_nofpu_proxy_id: + save_fpu_registers = false; + // fall through + case monitorexit_proxy_id: + { + StubFrame f(sasm, "monitorexit", dont_gc_arguments); + OopMap* map = save_live_registers(sasm, 2, save_fpu_registers); + // Called with store_parameter and not C abi + f.load_argument(0, rax); // rax,: lock address + int call_offset; + call_offset = __ call_RT(noreg, noreg, CAST_FROM_FN_PTR(address, monitorexit_wisp_proxy), rax); + oop_maps = new OopMapSet(); + oop_maps->add_gc_map(call_offset, map); + restore_live_registers(sasm, save_fpu_registers); + } + break; case deoptimize_id: { StubFrame f(sasm, "deoptimize", dont_gc_arguments); diff --git a/src/hotspot/cpu/x86/c2_MacroAssembler_x86.cpp b/src/hotspot/cpu/x86/c2_MacroAssembler_x86.cpp index 8f16747b6e1..9bc73c386d2 100644 --- a/src/hotspot/cpu/x86/c2_MacroAssembler_x86.cpp +++ b/src/hotspot/cpu/x86/c2_MacroAssembler_x86.cpp @@ -31,6 +31,7 @@ #include "opto/opcodes.hpp" #include "opto/subnode.hpp" #include "runtime/biasedLocking.hpp" +#include "runtime/coroutine.hpp" #include "runtime/objectMonitor.hpp" #include "runtime/stubRoutines.hpp" @@ -597,8 +598,17 @@ void C2_MacroAssembler::fast_lock(Register objReg, Register boxReg, Register tmp // It's inflated and we use scrReg for ObjectMonitor* in this section. movq(scrReg, tmpReg); xorq(tmpReg, tmpReg); + + // It's inflated and appears unlocked + if (UseWispMonitor) { + movptr (r15_thread, Address(r15_thread, JavaThread::current_coroutine_offset())); + movptr (r15_thread, Address(r15_thread, Coroutine::wisp_thread_offset())); + } lock(); cmpxchgptr(r15_thread, Address(scrReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner))); + if (UseWispMonitor) { + movptr (r15_thread, Address(r15_thread, WispThread::thread_offset())); + } // Unconditionally set box->_displaced_header = markWord::unused_mark(). // Without cast to int32_t this style of movptr will destroy r10 which is typically obj. movptr(Address(boxReg, 0), (int32_t)intptr_t(markWord::unused_mark().value())); @@ -688,7 +698,12 @@ void C2_MacroAssembler::fast_unlock(Register objReg, Register boxReg, Register t jcc (Assembler::zero, DONE_LABEL); // 0 indicates recursive stack-lock movptr(tmpReg, Address(objReg, oopDesc::mark_offset_in_bytes())); // Examine the object's markword testptr(tmpReg, markWord::monitor_value); // Inflated? - jccb (Assembler::zero, Stacked); + // If UseWispMonitor is enable, insert more code, then the length in jccb isn't enough. + if (UseWispMonitor) { + jcc (Assembler::zero, Stacked); + } else { + jccb (Assembler::zero, Stacked); + } // It's inflated. #if INCLUDE_RTM_OPT @@ -814,8 +829,15 @@ void C2_MacroAssembler::fast_unlock(Register objReg, Register boxReg, Register t // box is really RAX -- the following CMPXCHG depends on that binding // cmpxchg R,[M] is equivalent to rax = CAS(M,rax,R) + if (UseWispMonitor) { + movptr (r15_thread, Address(r15_thread, JavaThread::current_coroutine_offset())); + movptr (r15_thread, Address(r15_thread, Coroutine::wisp_thread_offset())); + } lock(); cmpxchgptr(r15_thread, Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner))); + if (UseWispMonitor) { + movptr (r15_thread, Address(r15_thread, WispThread::thread_offset())); + } // There's no successor so we tried to regrab the lock. // If that didn't work, then another thread grabbed the // lock so we're done (and exit was a success). diff --git a/src/hotspot/cpu/x86/interp_masm_x86.cpp b/src/hotspot/cpu/x86/interp_masm_x86.cpp index d92885daeca..16550f78bd4 100644 --- a/src/hotspot/cpu/x86/interp_masm_x86.cpp +++ b/src/hotspot/cpu/x86/interp_masm_x86.cpp @@ -806,6 +806,7 @@ void InterpreterMacroAssembler::jump_from_interpreted(Register method, Register if (JvmtiExport::can_post_interpreter_events()) { Label run_compiled_code; + Label coroutine_skip_interpret; // JVMTI events, such as single-stepping, are implemented partly by avoiding running // compiled code in threads for which the event is enabled. Check here for // interp_only_mode if these events CAN be enabled. @@ -815,8 +816,19 @@ void InterpreterMacroAssembler::jump_from_interpreted(Register method, Register NOT_LP64(get_thread(temp);) cmpb(Address(temp, JavaThread::interp_only_mode_offset()), 0); jccb(Assembler::zero, run_compiled_code); + if (EnableCoroutine) { + cmpq(Address(method, Method::intrinsic_id_offset_in_bytes()), (int)vmIntrinsics::_switchTo); + jcc(Assembler::zero, coroutine_skip_interpret); + cmpq(Address(method, Method::intrinsic_id_offset_in_bytes()), (int)vmIntrinsics::_switchToAndExit); + jcc(Assembler::zero, coroutine_skip_interpret); + cmpq(Address(method, Method::intrinsic_id_offset_in_bytes()), (int)vmIntrinsics::_switchToAndTerminate); + jcc(Assembler::zero, coroutine_skip_interpret); + } jmp(Address(method, Method::interpreter_entry_offset())); bind(run_compiled_code); + if (EnableCoroutine) { + bind(coroutine_skip_interpret); + } } jmp(Address(method, Method::from_interpreted_offset())); diff --git a/src/hotspot/cpu/x86/macroAssembler_x86.cpp b/src/hotspot/cpu/x86/macroAssembler_x86.cpp index 740189c144f..7fb116aec13 100644 --- a/src/hotspot/cpu/x86/macroAssembler_x86.cpp +++ b/src/hotspot/cpu/x86/macroAssembler_x86.cpp @@ -41,6 +41,7 @@ #include "oops/klass.inline.hpp" #include "prims/methodHandles.hpp" #include "runtime/biasedLocking.hpp" +#include "runtime/coroutine.hpp" #include "runtime/flags/flagSetting.hpp" #include "runtime/interfaceSupport.inline.hpp" #include "runtime/jniHandles.hpp" @@ -1278,6 +1279,11 @@ void MacroAssembler::bang_stack_size(Register size, Register tmp) { } void MacroAssembler::reserved_stack_check() { + //TODO: JDK11 introduces the usage of @ReservedStackAccess, we will support it with Coroutine + // If we check it in wisp flow, the exception is thrown when coroutine switches. + if (EnableCoroutine) { + return; + } // testing if reserved zone needs to be enabled Label no_reserved_zone_enabling; Register thread = NOT_LP64(rsi) LP64_ONLY(r15_thread); @@ -1693,6 +1699,8 @@ void MacroAssembler::super_call_VM(Register oop_result, super_call_VM(oop_result, last_java_sp, entry_point, 3, check_exceptions); } +extern address monitorenter_address_interp; + void MacroAssembler::call_VM_base(Register oop_result, Register java_thread, Register last_java_sp, @@ -1738,6 +1746,13 @@ void MacroAssembler::call_VM_base(Register oop_result, // do the call, remove parameters MacroAssembler::call_VM_leaf_base(entry_point, number_of_arguments); + if (EnableSteal) { + // only if the entry_point is equal to `Interpreter::monitorenter()`, we will do this amendment. + if (entry_point == monitorenter_address_interp) { + WISP_V2v_UPDATE; + } + } + // restore the thread (cannot use the pushed argument since arguments // may be overwritten by C code generated by an optimizing compiler); // however can use the register value directly if it is callee saved. @@ -1893,6 +1908,16 @@ void MacroAssembler::get_vm_result_2(Register metadata_result, Register java_thr movptr(Address(java_thread, JavaThread::vm_result_2_offset()), NULL_WORD); } +void MacroAssembler::get_vm_result_for_wisp(Register oop_result, Register java_thread) { + assert(EnableCoroutine, "Coroutine is disabled"); + movptr(oop_result, Address(java_thread, JavaThread::vm_result_for_wisp_offset())); + movptr(Address(java_thread, JavaThread::vm_result_for_wisp_offset()), NULL_WORD); + // In default flow, after calling get_vm_result, vm_result is set to null. + // In wisp flow, we should also clear vm_result. + movptr(Address(java_thread, JavaThread::vm_result_offset()), NULL_WORD); + verify_oop_msg(oop_result, "broken oop in call_VM_base"); +} + void MacroAssembler::check_and_handle_earlyret(Register java_thread) { } diff --git a/src/hotspot/cpu/x86/macroAssembler_x86.hpp b/src/hotspot/cpu/x86/macroAssembler_x86.hpp index fcde34b2751..1c65e35c529 100644 --- a/src/hotspot/cpu/x86/macroAssembler_x86.hpp +++ b/src/hotspot/cpu/x86/macroAssembler_x86.hpp @@ -274,6 +274,7 @@ class MacroAssembler: public Assembler { void get_vm_result (Register oop_result, Register thread); void get_vm_result_2(Register metadata_result, Register thread); + void get_vm_result_for_wisp(Register oop_result, Register thread); // These always tightly bind to MacroAssembler::call_VM_base // bypassing the virtual implementation diff --git a/src/hotspot/cpu/x86/sharedRuntime_x86_64.cpp b/src/hotspot/cpu/x86/sharedRuntime_x86_64.cpp index 25263f1ca1a..649a4dec9cf 100644 --- a/src/hotspot/cpu/x86/sharedRuntime_x86_64.cpp +++ b/src/hotspot/cpu/x86/sharedRuntime_x86_64.cpp @@ -1533,6 +1533,19 @@ static void gen_special_dispatch(MacroAssembler* masm, void create_switchTo_contents(MacroAssembler *masm, int start, OopMapSet* oop_maps, int &stack_slots, int total_in_args, BasicType *in_sig_bt, VMRegPair *in_regs, BasicType ret_type, bool terminate); +void generate_thread_fix(MacroAssembler *masm, Method *method) { + // we can have a check here at the codegen time, so no cost in runtime. + if (EnableSteal) { + if (WispStealCandidate(method->method_holder()->name(), method->name(), method->signature()).is_steal_candidate()) { + // as X86 calling conventions, the thread register r15 is one of the callee saved registers. + // see: https://www.cs.cmu.edu/~aplatzer/course/Compilers11/calling_conventions.pdf, callee saved registers + // the r15 register will be saved at method entry, and restored implicitly at method exit + // so we have to fix it manually here after the method returns. + WISP_X86_CONVENTION_V2J_UPDATE; + } + } +} + // --------------------------------------------------------------------------- // Generate a native wrapper for a given method. The method takes arguments // in the Java compiled code convention, marshals them to the native @@ -2142,6 +2155,11 @@ nmethod* SharedRuntime::generate_native_wrapper(MacroAssembler* masm, __ call(RuntimeAddress(native_func)); + // In wisp, this coroutine may be stolen by another thread inside the `native_func` call. + // If this `native_func` is one of the several native functions we supported, + // we will add thread fix code for them. + generate_thread_fix(masm, method()); + // Verify or restore cpu control state after JNI call __ restore_cpu_control_state_after_jni(); @@ -2368,6 +2386,11 @@ nmethod* SharedRuntime::generate_native_wrapper(MacroAssembler* masm, __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::complete_monitor_locking_C), 3); restore_args(masm, total_c_args, c_arg, out_regs); + if (EnableSteal) { + // the r15 has been restored in restore_args so we need fix it. + WISP_COMPILER_RESTORE_FORCE_UPDATE; + } + #ifdef ASSERT { Label L; __ cmpptr(Address(r15_thread, in_bytes(Thread::pending_exception_offset())), (int32_t)NULL_WORD); @@ -4032,10 +4055,17 @@ void create_switchTo_contents(MacroAssembler *masm, int start, OopMapSet* oop_ma __ movptr(Address(old_coroutine, Coroutine::active_handles_offset()), temp); __ movptr(temp, Address(thread, Thread::metadata_handles_offset())); __ movptr(Address(old_coroutine, Coroutine::metadata_handles_offset()), temp); -#ifdef ASSERT + __ movptr(temp, Address(thread, JavaThread::last_Java_pc_offset())); + __ movptr(Address(old_coroutine, Coroutine::last_Java_pc_offset()), temp); + __ movptr(temp, Address(thread, JavaThread::last_Java_sp_offset())); + __ movptr(Address(old_coroutine, Coroutine::last_Java_sp_offset()), temp); + // __ movptr(temp, Address(thread, JavaThread::privileged_stack_top_offset())); + // __ movptr(Address(old_coroutine, Coroutine::privileged_stack_top_offset()), temp); + __ movptr(temp, Address(thread, JavaThread::threadObj_offset())); + __ movl(temp, Address(temp, java_lang_Thread::thread_status_offset())); + __ movl(Address(old_coroutine, Coroutine::thread_status_offset()), temp); __ movl(temp, Address(thread, JavaThread::java_call_counter_offset())); __ movl(Address(old_coroutine, Coroutine::java_call_counter_offset()), temp); -#endif __ movptr(Address(old_stack, CoroutineStack::last_sp_offset()), rsp); } @@ -4066,11 +4096,18 @@ void create_switchTo_contents(MacroAssembler *masm, int start, OopMapSet* oop_ma __ movptr(Address(thread, Thread::active_handles_offset()), temp); __ movptr(temp, Address(target_coroutine, Coroutine::metadata_handles_offset())); __ movptr(Address(thread, Thread::metadata_handles_offset()), temp); - -#ifdef ASSERT + __ movptr(temp, Address(target_coroutine, Coroutine::last_Java_pc_offset())); + __ movptr(Address(thread, JavaThread::last_Java_pc_offset()), temp); + __ movptr(temp, Address(target_coroutine, Coroutine::last_Java_sp_offset())); + __ movptr(Address(thread, JavaThread::last_Java_sp_offset()), temp); + // __ movptr(temp, Address(target_coroutine, Coroutine::privileged_stack_top_offset())); + // __ movptr(Address(thread, JavaThread::privileged_stack_top_offset()), temp); + __ movl(temp2, Address(target_coroutine, Coroutine::thread_status_offset())); + __ movptr(temp, Address(thread, JavaThread::threadObj_offset())); + __ movl(Address(temp, java_lang_Thread::thread_status_offset()), temp2); __ movl(temp, Address(target_coroutine, Coroutine::java_call_counter_offset())); __ movl(Address(thread, JavaThread::java_call_counter_offset()), temp); - +#ifdef ASSERT __ movptr(Address(target_coroutine, Coroutine::handle_area_offset()), (intptr_t)NULL_WORD); __ movptr(Address(target_coroutine, Coroutine::resource_area_offset()), (intptr_t)NULL_WORD); __ movptr(Address(target_coroutine, Coroutine::last_handle_mark_offset()), (intptr_t)NULL_WORD); diff --git a/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp b/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp index c7634a2e596..54dfaeae2f3 100644 --- a/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp +++ b/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp @@ -40,6 +40,7 @@ #include "oops/oop.inline.hpp" #include "prims/methodHandles.hpp" #include "runtime/arguments.hpp" +#include "runtime/coroutine.hpp" #include "runtime/frame.inline.hpp" #include "runtime/handles.inline.hpp" #include "runtime/sharedRuntime.hpp" @@ -363,6 +364,14 @@ class StubGenerator: public StubCodeGenerator { // pop parameters __ lea(rsp, rsp_after_call); + if (EnableSteal) { + Label L; // the steal thread patch + __ cmpptr(r15_thread, thread); + __ jcc(Assembler::equal, L); + WISP_j2v_UPDATE; + __ bind(L); + } + #ifdef ASSERT // verify that threads correspond { @@ -453,6 +462,11 @@ class StubGenerator: public StubCodeGenerator { const Address rsp_after_call(rbp, rsp_after_call_off * wordSize); const Address thread (rbp, thread_off * wordSize); + if (EnableSteal) { + // to pass the assertion fail + WISP_j2v_UPDATE; + } + #ifdef ASSERT // verify that threads correspond { diff --git a/src/hotspot/cpu/x86/templateInterpreterGenerator_x86.cpp b/src/hotspot/cpu/x86/templateInterpreterGenerator_x86.cpp index dcbb3733390..506f49e9f20 100644 --- a/src/hotspot/cpu/x86/templateInterpreterGenerator_x86.cpp +++ b/src/hotspot/cpu/x86/templateInterpreterGenerator_x86.cpp @@ -40,6 +40,7 @@ #include "oops/oop.inline.hpp" #include "prims/jvmtiExport.hpp" #include "prims/jvmtiThreadState.hpp" +#include "runtime/coroutine.hpp" #include "runtime/deoptimization.hpp" #include "runtime/frame.inline.hpp" #include "runtime/jniHandles.hpp" @@ -998,6 +999,13 @@ address TemplateInterpreterGenerator::generate_native_entry(bool synchronized) { // 32: result potentially in rdx:rax or ST0 // 64: result potentially in rax or xmm0 + // when we get back from native c++ code, our coroutine may be stolen by another thread. + if (EnableSteal) { + // it is a little hard to present method signature check here: because interpreter + // will directly jump into this entry, which is in runtime. + WISP_X86_CONVENTION_V2j_UPDATE; + } + // Verify or restore cpu control state after JNI call __ restore_cpu_control_state_after_jni(); @@ -1626,12 +1634,32 @@ void TemplateInterpreterGenerator::generate_throw_exception() { // preserve exception over this code sequence __ pop_ptr(rax); NOT_LP64(__ get_thread(thread)); - __ movptr(Address(thread, JavaThread::vm_result_offset()), rax); + if (EnableCoroutine && UseWispMonitor) { + /** + * this fix is added here due to the fact that this function may call back to java(at _monitorexit when UseWispMonitor is enabled) + * Generally, _monitorexit is invoked by: 1. _return bytecode 2. when exception happens, 3. normally unlock an object. + * 1 and 2 will call remove_activation to force unwinding current stack and go up to the caller. + * When exception happens, program will run to: _remove_activation_entry and then remove_activation. + * they both will get call here while facing race condition. + * Only when 2 happens, hotspot will save the exception oop into rax, then call remove_activation. + * But because remove_activation possibly use rax, it hence saves rax (exception oop) to thread->_vm_result temporarily. + * It won't see the problem in normal case because _monitorexit won't call to java(the original vm code impl. does get chance to modify _vm_result) + * Because of the call back to java required by Wisp impl. at _monitorexit which in turn may modify _vm_result, + * The current fix MUST to restore the exception oop into different field(_vm_result_for_wisp ). + */ + __ movptr(Address(thread, JavaThread::vm_result_for_wisp_offset()), rax); + } else { + __ movptr(Address(thread, JavaThread::vm_result_offset()), rax); + } // remove the activation (without doing throws on illegalMonitorExceptions) __ remove_activation(vtos, rdx, false, true, false); // restore exception NOT_LP64(__ get_thread(thread)); - __ get_vm_result(rax, thread); + if (EnableCoroutine && UseWispMonitor) { + __ get_vm_result_for_wisp(rax, thread); + } else { + __ get_vm_result(rax, thread); + } // In between activations - previous activation type unknown yet // compute continuation point - the continuation point expects the diff --git a/src/hotspot/cpu/x86/x86.ad b/src/hotspot/cpu/x86/x86.ad index 57e22fe79aa..48a2bd7d3d6 100644 --- a/src/hotspot/cpu/x86/x86.ad +++ b/src/hotspot/cpu/x86/x86.ad @@ -1162,6 +1162,7 @@ source_hpp %{ // we switch between source %{ }% and source_hpp %{ }% freely as needed. #include "runtime/vm_version.hpp" +#include "runtime/coroutine.hpp" class NativeJump; diff --git a/src/hotspot/cpu/x86/x86_64.ad b/src/hotspot/cpu/x86/x86_64.ad index 3a95006819a..a07917c0cc0 100644 --- a/src/hotspot/cpu/x86/x86_64.ad +++ b/src/hotspot/cpu/x86/x86_64.ad @@ -13156,6 +13156,20 @@ instruct tlsLoadP(r15_RegP dst) %{ ins_pipe(ialu_reg_reg); %} +// Thread refetch: +// take two main arguments: +// 1. register @r15 +// 2. one register which contains the `Coroutine *` +// and move Coroutine->_thread to @r15 +instruct tlsRefetchP(r15_RegP dst, any_RegP src) %{ + match(Set dst (ThreadRefetch src)); + + format %{ "Refetch the r15 register" %} + ins_encode %{ + __ movptr(r15, Address($src$$Register, Coroutine::thread_offset())); + %} + ins_pipe(ialu_reg_reg); +%} //----------PEEPHOLE RULES----------------------------------------------------- // These must follow all instruction definitions as they use the names diff --git a/src/hotspot/os/posix/os_posix.cpp b/src/hotspot/os/posix/os_posix.cpp index 5c34a30a07d..8bb484c4ac8 100644 --- a/src/hotspot/os/posix/os_posix.cpp +++ b/src/hotspot/os/posix/os_posix.cpp @@ -24,6 +24,7 @@ #include "jvm.h" +#include "memory/allocation.hpp" #ifdef LINUX #include "classfile/classLoader.hpp" #endif @@ -875,6 +876,19 @@ void os::naked_short_sleep(jlong ms) { return; } +bool clear_interrupt_for_wisp(Thread* thread) { + assert(EnableCoroutine, "Coroutine is disabled"); + // If we only use -XX:+EnableCoroutine and -Dcom.alibaba.transparentAsync=true, we will + // fall here, so we cannot use `assert(UseWispMonitor)` only. + if (UseWispMonitor && thread->is_Wisp_thread()) { + thread = ((WispThread *)thread)->thread(); + } + bool interrupted = ((JavaThread*) thread->as_Java_thread())->is_interrupted(false); + thread->osthread()->set_interrupted(false); + + return interrupted; +} + char* os::Posix::describe_pthread_attr(char* buf, size_t buflen, const pthread_attr_t* attr) { size_t stack_size = 0; size_t guard_size = 0; diff --git a/src/hotspot/share/c1/c1_CodeStubs.hpp b/src/hotspot/share/c1/c1_CodeStubs.hpp index 961f33ca69e..6dc31f52aba 100644 --- a/src/hotspot/share/c1/c1_CodeStubs.hpp +++ b/src/hotspot/share/c1/c1_CodeStubs.hpp @@ -365,11 +365,22 @@ class MonitorExitStub: public MonitorAccessStub { private: bool _compute_lock; int _monitor_ix; - + CodeEmitInfo* _info; public: MonitorExitStub(LIR_Opr lock_reg, bool compute_lock, int monitor_ix) : MonitorAccessStub(LIR_OprFact::illegalOpr, lock_reg), _compute_lock(compute_lock), _monitor_ix(monitor_ix) { } + // This constructor is used in Wisp only. Generating CodeEmitInfo for furter use. + MonitorExitStub(LIR_Opr lock_reg, bool compute_lock, int monitor_ix, CodeEmitInfo* info) + : MonitorAccessStub(LIR_OprFact::illegalOpr, lock_reg), + _compute_lock(compute_lock), _monitor_ix(monitor_ix) { + if (info) { + assert(UseWispMonitor, "This path should be reached only when using WispMonitor"); + _info = new CodeEmitInfo(info); + } else { + _info = NULL; + } + } virtual void emit_code(LIR_Assembler* e); virtual void visit(LIR_OpVisitState* visitor) { assert(_obj_reg->is_illegal(), "unused"); @@ -378,6 +389,9 @@ class MonitorExitStub: public MonitorAccessStub { } else { visitor->do_input(_lock_reg); } + if (UseWispMonitor && _info != NULL) { + visitor->do_slow_case(_info); + } } #ifndef PRODUCT virtual void print_name(outputStream* out) const { out->print("MonitorExitStub"); } diff --git a/src/hotspot/share/c1/c1_GraphBuilder.cpp b/src/hotspot/share/c1/c1_GraphBuilder.cpp index 3c2545ec661..6cc436852a0 100644 --- a/src/hotspot/share/c1/c1_GraphBuilder.cpp +++ b/src/hotspot/share/c1/c1_GraphBuilder.cpp @@ -1554,7 +1554,13 @@ void GraphBuilder::method_return(Value x, bool ignore_return) { // released before we jump to the continuation block. if (method()->is_synchronized()) { assert(state()->locks_size() == 1, "receiver must be locked here"); - monitorexit(state()->lock_at(0), SynchronizationEntryBCI); + if (UseWispMonitor) { + // It is needed to distinguish monitorexit called at method return from other locations + // Because it is unnecessary to generate oopmap at method return. + monitorexit(state()->lock_at(0), SynchronizationEntryBCI, true); + } else { + monitorexit(state()->lock_at(0), SynchronizationEntryBCI); + } } if (need_mem_bar) { @@ -1598,7 +1604,7 @@ void GraphBuilder::method_return(Value x, bool ignore_return) { } else { receiver = append(new Constant(new ClassConstant(method()->holder()))); } - append_split(new MonitorExit(receiver, state()->unlock())); + append_split(new MonitorExit(receiver, state()->unlock(), NULL)); } if (need_mem_bar) { @@ -2246,6 +2252,20 @@ void GraphBuilder::monitorexit(Value x, int bci) { kill_all(); } +void GraphBuilder::monitorexit(Value x, int bci, bool at_method_return) { + if (UseWispMonitor) { + if (at_method_return == false) { + ValueStack* state_before = copy_state_for_exception_with_bci(bci); + Instruction* instr = new MonitorExit(x, state()->unlock(), state_before); + append_with_bci(instr, bci); + } else { + append_with_bci(new MonitorExit(x, state()->unlock()), bci); + } + } else { + append_with_bci(new MonitorExit(x, state()->unlock()), bci); + } + kill_all(); +} void GraphBuilder::new_multi_array(int dimensions) { ciKlass* klass = stream()->get_klass(); diff --git a/src/hotspot/share/c1/c1_GraphBuilder.hpp b/src/hotspot/share/c1/c1_GraphBuilder.hpp index 80058a210a8..3d26e10817c 100644 --- a/src/hotspot/share/c1/c1_GraphBuilder.hpp +++ b/src/hotspot/share/c1/c1_GraphBuilder.hpp @@ -263,6 +263,7 @@ class GraphBuilder { void instance_of(int klass_index); void monitorenter(Value x, int bci); void monitorexit(Value x, int bci); + void monitorexit(Value x, int bci, bool at_method_return); void new_multi_array(int dimensions); void throw_op(int bci); Value round_fp(Value fp_value); diff --git a/src/hotspot/share/c1/c1_Instruction.hpp b/src/hotspot/share/c1/c1_Instruction.hpp index 858d4f606ed..fbb46be35d3 100644 --- a/src/hotspot/share/c1/c1_Instruction.hpp +++ b/src/hotspot/share/c1/c1_Instruction.hpp @@ -1514,6 +1514,12 @@ LEAF(MonitorExit, AccessMonitor) { ASSERT_VALUES } + + MonitorExit(Value obj, int monitor_no, ValueStack* state_before) + : AccessMonitor(obj, monitor_no, state_before) + { + ASSERT_VALUES + } }; diff --git a/src/hotspot/share/c1/c1_LIR.cpp b/src/hotspot/share/c1/c1_LIR.cpp index 62cff4c7505..19c50371368 100644 --- a/src/hotspot/share/c1/c1_LIR.cpp +++ b/src/hotspot/share/c1/c1_LIR.cpp @@ -1359,7 +1359,7 @@ void LIR_List::lock_object(LIR_Opr hdr, LIR_Opr obj, LIR_Opr lock, LIR_Opr scrat info)); } -void LIR_List::unlock_object(LIR_Opr hdr, LIR_Opr obj, LIR_Opr lock, LIR_Opr scratch, CodeStub* stub) { +void LIR_List::unlock_object(LIR_Opr hdr, LIR_Opr obj, LIR_Opr lock, LIR_Opr scratch, CodeStub* stub, CodeEmitInfo* info) { append(new LIR_OpLock( lir_unlock, hdr, @@ -1367,7 +1367,7 @@ void LIR_List::unlock_object(LIR_Opr hdr, LIR_Opr obj, LIR_Opr lock, LIR_Opr scr lock, scratch, stub, - NULL)); + info)); } diff --git a/src/hotspot/share/c1/c1_LIR.hpp b/src/hotspot/share/c1/c1_LIR.hpp index 2342e6117eb..3dd586e7bf9 100644 --- a/src/hotspot/share/c1/c1_LIR.hpp +++ b/src/hotspot/share/c1/c1_LIR.hpp @@ -2223,7 +2223,7 @@ class LIR_List: public CompilationResourceObj { } void load_stack_address_monitor(int monitor_ix, LIR_Opr dst) { append(new LIR_Op1(lir_monaddr, LIR_OprFact::intConst(monitor_ix), dst)); } - void unlock_object(LIR_Opr hdr, LIR_Opr obj, LIR_Opr lock, LIR_Opr scratch, CodeStub* stub); + void unlock_object(LIR_Opr hdr, LIR_Opr obj, LIR_Opr lock, LIR_Opr scratch, CodeStub* stub, CodeEmitInfo* info = NULL); void lock_object(LIR_Opr hdr, LIR_Opr obj, LIR_Opr lock, LIR_Opr scratch, CodeStub* stub, CodeEmitInfo* info); void breakpoint() { append(new LIR_Op0(lir_breakpoint)); } diff --git a/src/hotspot/share/c1/c1_LIRGenerator.cpp b/src/hotspot/share/c1/c1_LIRGenerator.cpp index e5e71fab8c1..faa6e0119f2 100644 --- a/src/hotspot/share/c1/c1_LIRGenerator.cpp +++ b/src/hotspot/share/c1/c1_LIRGenerator.cpp @@ -425,7 +425,7 @@ CodeEmitInfo* LIRGenerator::state_for(Instruction* x, ValueStack* state, bool ig // all locals are dead on exit from the synthetic unlocker liveness.clear(); } else { - assert(x->as_MonitorEnter() || x->as_ProfileInvoke(), "only other cases are MonitorEnter and ProfileInvoke"); + assert(x->as_MonitorEnter() || x->as_ProfileInvoke() || (UseWispMonitor && x->as_MonitorExit()), "only other cases are MonitorEnter and ProfileInvoke, or Wisp MonitorExit"); } } if (!liveness.is_valid()) { @@ -631,14 +631,21 @@ void LIRGenerator::monitor_enter(LIR_Opr object, LIR_Opr lock, LIR_Opr hdr, LIR_ } -void LIRGenerator::monitor_exit(LIR_Opr object, LIR_Opr lock, LIR_Opr new_hdr, LIR_Opr scratch, int monitor_no) { +void LIRGenerator::monitor_exit(LIR_Opr object, LIR_Opr lock, LIR_Opr new_hdr, LIR_Opr scratch, int monitor_no, CodeEmitInfo* info_for_exception, CodeEmitInfo* info) { if (!GenerateSynchronizationCode) return; // setup registers LIR_Opr hdr = lock; lock = new_hdr; - CodeStub* slow_path = new MonitorExitStub(lock, UseFastLocking, monitor_no); - __ load_stack_address_monitor(monitor_no, lock); - __ unlock_object(hdr, object, lock, scratch, slow_path); + CodeStub* slow_path; + if (UseWispMonitor) { + slow_path = new MonitorExitStub(lock, UseFastLocking, monitor_no, info); + __ load_stack_address_monitor(monitor_no, lock); + __ unlock_object(hdr, object, lock, scratch, slow_path, info_for_exception); + } else { + slow_path = new MonitorExitStub(lock, UseFastLocking, monitor_no); + __ load_stack_address_monitor(monitor_no, lock); + __ unlock_object(hdr, object, lock, scratch, slow_path); + } } #ifndef PRODUCT diff --git a/src/hotspot/share/c1/c1_LIRGenerator.hpp b/src/hotspot/share/c1/c1_LIRGenerator.hpp index 67c986cb4e7..67e7a719aeb 100644 --- a/src/hotspot/share/c1/c1_LIRGenerator.hpp +++ b/src/hotspot/share/c1/c1_LIRGenerator.hpp @@ -360,7 +360,7 @@ class LIRGenerator: public InstructionVisitor, public BlockClosure { void logic_op (Bytecodes::Code code, LIR_Opr dst_reg, LIR_Opr left, LIR_Opr right); void monitor_enter (LIR_Opr object, LIR_Opr lock, LIR_Opr hdr, LIR_Opr scratch, int monitor_no, CodeEmitInfo* info_for_exception, CodeEmitInfo* info); - void monitor_exit (LIR_Opr object, LIR_Opr lock, LIR_Opr hdr, LIR_Opr scratch, int monitor_no); + void monitor_exit (LIR_Opr object, LIR_Opr lock, LIR_Opr hdr, LIR_Opr scratch, int monitor_no, CodeEmitInfo* info_for_exception = NULL, CodeEmitInfo* info = NULL); void new_instance (LIR_Opr dst, ciInstanceKlass* klass, bool is_unresolved, LIR_Opr scratch1, LIR_Opr scratch2, LIR_Opr scratch3, LIR_Opr scratch4, LIR_Opr klass_reg, CodeEmitInfo* info); diff --git a/src/hotspot/share/c1/c1_Runtime1.cpp b/src/hotspot/share/c1/c1_Runtime1.cpp index fb178432d8b..351204ac829 100644 --- a/src/hotspot/share/c1/c1_Runtime1.cpp +++ b/src/hotspot/share/c1/c1_Runtime1.cpp @@ -697,22 +697,68 @@ JRT_ENTRY(void, Runtime1::throw_incompatible_class_change_error(JavaThread* curr JRT_END +// funtions in `Runtime1` are all private, so add a function pointer to get its address +void (*Runtime1::monitorenter_address_C1)(JavaThread *, oopDesc* obj, BasicObjectLock *) = Runtime1::monitorenter; +address monitorenter_address_C1 = (address)Runtime1::monitorenter_address_C1; + + JRT_BLOCK_ENTRY(void, Runtime1::monitorenter(JavaThread* current, oopDesc* obj, BasicObjectLock* lock)) NOT_PRODUCT(_monitorenter_slowcase_cnt++;) if (!UseFastLocking) { lock->set_obj(obj); } assert(obj == lock->obj(), "must match"); + WispPostStealHandleUpdateMark w(__hm); SharedRuntime::monitor_enter_helper(obj, lock->lock(), current); JRT_END +JRT_ENTRY_NO_ASYNC(void, Runtime1::monitorexit_wisp(JavaThread* current, BasicObjectLock* lock)) + NOT_PRODUCT(_monitorexit_slowcase_cnt++;) + assert(UseWispMonitor, "UseWispMonitor is off"); + JavaThread* thread_tmp = NULL; + ExceptionMark __em(thread_tmp); + oop obj = lock->obj(); + // Almost a copy from Runtime1::monitorexit, + // excpet that handles are used to access objects. + Handle h_obj(current, obj); + ObjectSynchronizer::exit(h_obj, lock->lock(), THREAD); +JRT_END + + +// Handle spcecial case for wisp unpark. +// This function is executed only when the following four conditions are all satisfied +// 1. A synchronized method is compiled by C1 +// 2. An exception happened in this method +// 3. There is no exception handler in this method, So it needs to unwind to its caller +// 4. GC happened during unpark +// This path will not call Java, so JRT_LEAF is used. +JRT_LEAF(void, Runtime1::monitorexit_wisp_proxy(JavaThread* thread, BasicObjectLock* lock)) + NOT_PRODUCT(_monitorexit_slowcase_cnt++;) + assert(UseWispMonitor, "UseWispMonitor is off"); + EXCEPTION_MARK; + oop obj = lock->obj(); + assert(oopDesc::is_oop(obj), "must be NULL or an object"); + // Setting _is_proxy_unpark of current wisp thread to true. + // Proxy unpark will be used when this flag is true. + WispThread* wisp_thread = WispThread::current(thread); + wisp_thread->set_proxy_unpark_flag(); + // When using fast locking, the compiled code has already tried the fast case + ObjectSynchronizer::exit(obj, lock->lock(), THREAD); +JRT_END + + JRT_LEAF(void, Runtime1::monitorexit(JavaThread* current, BasicObjectLock* lock)) NOT_PRODUCT(_monitorexit_slowcase_cnt++;) assert(current->last_Java_sp(), "last_Java_sp must be set"); oop obj = lock->obj(); assert(oopDesc::is_oop(obj), "must be NULL or an object"); - SharedRuntime::monitor_exit_helper(obj, lock->lock(), current); + if (UseWispMonitor) { + HandleMarkCleaner __hm(current); + SharedRuntime::monitor_exit_helper(obj, lock->lock(), current); + } else { + SharedRuntime::monitor_exit_helper(obj, lock->lock(), current); + } JRT_END // Cf. OptoRuntime::deoptimize_caller_frame diff --git a/src/hotspot/share/c1/c1_Runtime1.hpp b/src/hotspot/share/c1/c1_Runtime1.hpp index 9212a4fb249..c9408913e3a 100644 --- a/src/hotspot/share/c1/c1_Runtime1.hpp +++ b/src/hotspot/share/c1/c1_Runtime1.hpp @@ -63,6 +63,8 @@ class StubAssembler; stub(monitorenter_nofpu) /* optimized version that does not preserve fpu registers */ \ stub(monitorexit) \ stub(monitorexit_nofpu) /* optimized version that does not preserve fpu registers */ \ + stub(monitorexit_proxy) \ + stub(monitorexit_nofpu_proxy) /* optimized version that does not preserve fpu registers */ \ stub(deoptimize) \ stub(access_field_patching) \ stub(load_klass_patching) \ @@ -92,6 +94,9 @@ class Runtime1: public AllStatic { RUNTIME1_STUBS(DECLARE_STUB_ID, DECLARE_LAST_STUB_ID) }; + // only a function pointer + static void (*monitorenter_address_C1)(JavaThread *, oopDesc* obj, BasicObjectLock *); + // statistics #ifndef PRODUCT static int _generic_arraycopystub_cnt; @@ -155,6 +160,8 @@ class Runtime1: public AllStatic { static void monitorenter(JavaThread* current, oopDesc* obj, BasicObjectLock* lock); static void monitorexit (JavaThread* current, BasicObjectLock* lock); + static void monitorexit_wisp (JavaThread* current, BasicObjectLock* lock); + static void monitorexit_wisp_proxy (JavaThread* current, BasicObjectLock* lock); static void deoptimize(JavaThread* current, jint trap_request); static int access_field_patching(JavaThread* current); diff --git a/src/hotspot/share/classfile/dictionary.cpp b/src/hotspot/share/classfile/dictionary.cpp index 9905eb4fc70..abc827a8bd4 100644 --- a/src/hotspot/share/classfile/dictionary.cpp +++ b/src/hotspot/share/classfile/dictionary.cpp @@ -423,7 +423,7 @@ void Dictionary::validate_protection_domain(unsigned int name_hash, // security manager is installed later. Calls to load the same class with class loader // and protection domain are expected to succeed. { - MutexLocker mu(THREAD, SystemDictionary_lock); + SystemDictLocker mu(THREAD, SystemDictionary_lock); int d_index = hash_to_index(name_hash); add_protection_domain(d_index, name_hash, klass, protection_domain); diff --git a/src/hotspot/share/classfile/javaClasses.cpp b/src/hotspot/share/classfile/javaClasses.cpp index a11349e8b97..e0802196c1a 100644 --- a/src/hotspot/share/classfile/javaClasses.cpp +++ b/src/hotspot/share/classfile/javaClasses.cpp @@ -1760,6 +1760,7 @@ int java_lang_Thread::_stackSize_offset; int java_lang_Thread::_tid_offset; int java_lang_Thread::_thread_status_offset; int java_lang_Thread::_park_blocker_offset; +int java_lang_Thread::_park_event_offset; #define THREAD_FIELDS_DO(macro) \ macro(_name_offset, k, vmSymbols::name_name(), string_signature, false); \ @@ -1774,7 +1775,8 @@ int java_lang_Thread::_park_blocker_offset; macro(_stackSize_offset, k, "stackSize", long_signature, false); \ macro(_tid_offset, k, "tid", long_signature, false); \ macro(_thread_status_offset, k, "threadStatus", int_signature, false); \ - macro(_park_blocker_offset, k, "parkBlocker", object_signature, false) + macro(_park_blocker_offset, k, "parkBlocker", object_signature, false); \ + macro(_park_event_offset, k, "nativeParkEventPointer", long_signature, false) void java_lang_Thread::compute_offsets() { assert(_group_offset == 0, "offsets should be initialized only once"); @@ -1906,6 +1908,21 @@ oop java_lang_Thread::park_blocker(oop java_thread) { return java_thread->obj_field(_park_blocker_offset); } +jlong java_lang_Thread::park_event(oop java_thread) { + if (_park_event_offset > 0) { + return java_thread->long_field(_park_event_offset); + } + return 0; +} + +bool java_lang_Thread::set_park_event(oop java_thread, jlong ptr) { + if (_park_event_offset > 0) { + java_thread->long_field_put(_park_event_offset, ptr); + return true; + } + return false; +} + const char* java_lang_Thread::thread_status_name(oop java_thread) { JavaThreadStatus status = static_cast(java_thread->int_field(_thread_status_offset)); switch (status) { @@ -4735,6 +4752,71 @@ void java_dyn_CoroutineBase::set_data(oop obj, jlong value) { obj->long_field_put(_data_offset, value); } +int com_alibaba_wisp_engine_WispEngine::_isInCritical_offset = 0; + +void com_alibaba_wisp_engine_WispEngine::compute_offsets() { + Klass* k = vmClasses::com_alibaba_wisp_engine_WispEngine_klass(); + assert(k != NULL, "WispEngine_klass is null"); + compute_offset(_isInCritical_offset, InstanceKlass::cast(k), vmSymbols::isInCritical_name(), vmSymbols::bool_signature()); +} + +bool com_alibaba_wisp_engine_WispEngine::in_critical(oop obj) { + return obj->bool_field(_isInCritical_offset); +} + +int com_alibaba_wisp_engine_WispTask::_jvmParkStatus_offset = 0; +int com_alibaba_wisp_engine_WispTask::_id_offset = 0; +int com_alibaba_wisp_engine_WispTask::_threadWrapper_offset = 0; +int com_alibaba_wisp_engine_WispTask::_interrupted_offset = 0; +int com_alibaba_wisp_engine_WispTask::_activeCount_offset = 0; +int com_alibaba_wisp_engine_WispTask::_stealCount_offset = 0; +int com_alibaba_wisp_engine_WispTask::_stealFailureCount_offset = 0; + +void com_alibaba_wisp_engine_WispTask::compute_offsets() { + Klass* k = vmClasses::com_alibaba_wisp_engine_WispTask_klass(); + assert(k != NULL, "WispTask_klass is null"); + InstanceKlass *ik = InstanceKlass::cast(k); + compute_offset(_jvmParkStatus_offset, ik, vmSymbols::jvmParkStatus_name(), vmSymbols::int_signature()); + compute_offset(_id_offset, ik, vmSymbols::id_name(), vmSymbols::int_signature()); + compute_offset(_threadWrapper_offset, ik, vmSymbols::threadWrapper_name(), vmSymbols::thread_signature()); + compute_offset(_interrupted_offset, ik, vmSymbols::interrupted_name(), vmSymbols::int_signature()); + compute_offset(_activeCount_offset, ik, vmSymbols::activeCount_name(), vmSymbols::int_signature()); + compute_offset(_stealCount_offset, ik, vmSymbols::stealCount_name(), vmSymbols::int_signature()); + compute_offset(_stealFailureCount_offset, ik, vmSymbols::stealFailureCount_name(), vmSymbols::int_signature()); +} + +void com_alibaba_wisp_engine_WispTask::set_jvmParkStatus(oop obj, jint status) { + return obj->int_field_put(_jvmParkStatus_offset, status); +} + +int com_alibaba_wisp_engine_WispTask::get_id(oop obj) { + return obj->int_field(_id_offset); +} + +oop com_alibaba_wisp_engine_WispTask::get_threadWrapper(oop obj) { + return obj->obj_field(_threadWrapper_offset); +} + +int com_alibaba_wisp_engine_WispTask::get_interrupted(oop obj) { + return obj->int_field(_interrupted_offset); +} + +void com_alibaba_wisp_engine_WispTask::set_interrupted(oop obj, jint interrupted) { + obj->int_field_put(_interrupted_offset, interrupted); +} + +int com_alibaba_wisp_engine_WispTask::get_activeCount(oop obj) { + return obj->int_field(_activeCount_offset); +} + +int com_alibaba_wisp_engine_WispTask::get_stealCount(oop obj) { + return obj->int_field(_stealCount_offset); +} + +int com_alibaba_wisp_engine_WispTask::get_stealFailureCount(oop obj) { + return obj->int_field(_stealFailureCount_offset); +} + #if INCLUDE_CDS void java_nio_Buffer::serialize_offsets(SerializeClosure* f) { BUFFER_FIELDS_DO(FIELD_SERIALIZE_OFFSET); @@ -5068,6 +5150,8 @@ void java_lang_InternalError::serialize_offsets(SerializeClosure* f) { void JavaClasses::compute_offsets() { if (EnableCoroutine) { java_dyn_CoroutineBase::compute_offsets(); + com_alibaba_wisp_engine_WispEngine::compute_offsets(); + com_alibaba_wisp_engine_WispTask::compute_offsets(); } if (UseSharedSpaces) { diff --git a/src/hotspot/share/classfile/javaClasses.hpp b/src/hotspot/share/classfile/javaClasses.hpp index b23724b5341..6f9c823ce34 100644 --- a/src/hotspot/share/classfile/javaClasses.hpp +++ b/src/hotspot/share/classfile/javaClasses.hpp @@ -412,6 +412,7 @@ class java_lang_Thread : AllStatic { static int _tid_offset; static int _thread_status_offset; static int _park_blocker_offset; + static int _park_event_offset; static void compute_offsets(); @@ -454,6 +455,12 @@ class java_lang_Thread : AllStatic { // Blocker object responsible for thread parking static oop park_blocker(oop java_thread); + // Pointer to type-stable park handler, encoded as jlong. + // Should be set when apparently null + // For details, see unsafe.cpp Unsafe_Unpark + static jlong park_event(oop java_thread); + static bool set_park_event(oop java_thread, jlong ptr); + // Write thread status info to threadStatus field of java.lang.Thread. static void set_thread_status(oop java_thread_oop, JavaThreadStatus status); // Read thread status info from threadStatus field of java.lang.Thread. @@ -461,6 +468,8 @@ class java_lang_Thread : AllStatic { static const char* thread_status_name(oop java_thread_oop); + static int thread_status_offset() { return _thread_status_offset; } + // Debugging friend class JavaClasses; }; @@ -1800,6 +1809,37 @@ class java_dyn_CoroutineBase: AllStatic { friend class JavaClasses; }; +class com_alibaba_wisp_engine_WispEngine: AllStatic { +private: + static int _isInCritical_offset; +public: + static bool in_critical(oop obj); + + static void compute_offsets(); +}; + +class com_alibaba_wisp_engine_WispTask: AllStatic { +private: + static int _jvmParkStatus_offset; + static int _id_offset; + static int _threadWrapper_offset; + static int _interrupted_offset; + static int _activeCount_offset; + static int _stealCount_offset; + static int _stealFailureCount_offset; +public: + static void set_jvmParkStatus(oop obj, jint status); + static int get_id(oop obj); + static oop get_threadWrapper(oop obj); + static int get_interrupted(oop obj); + static void set_interrupted(oop obj, jint interrupted); + static int get_activeCount(oop obj); + static int get_stealCount(oop obj); + static int get_stealFailureCount(oop obj); + + static void compute_offsets(); +}; + // Interface to hard-coded offset checking class JavaClasses : AllStatic { diff --git a/src/hotspot/share/classfile/placeholders.cpp b/src/hotspot/share/classfile/placeholders.cpp index 88b8d0fcc20..9f698f287a8 100644 --- a/src/hotspot/share/classfile/placeholders.cpp +++ b/src/hotspot/share/classfile/placeholders.cpp @@ -29,6 +29,7 @@ #include "logging/logTag.hpp" #include "logging/logStream.hpp" #include "memory/resourceArea.hpp" +#include "runtime/coroutine.hpp" #include "runtime/mutexLocker.hpp" #include "runtime/thread.hpp" #include "utilities/hashtable.inline.hpp" @@ -109,7 +110,7 @@ void PlaceholderEntry::set_threadQ(SeenThread* seenthread, PlaceholderTable::cla // define token. Appends for debugging of requestor order void PlaceholderEntry::add_seen_thread(Thread* thread, PlaceholderTable::classloadAction action) { assert_lock_strong(SystemDictionary_lock); - SeenThread* threadEntry = new SeenThread(thread); + SeenThread* threadEntry = new SeenThread(UseWispMonitor ? WispThread::current(thread) : thread); SeenThread* seen = actionToQueue(action); assert(action != PlaceholderTable::LOAD_INSTANCE || seen == NULL, @@ -130,6 +131,9 @@ void PlaceholderEntry::add_seen_thread(Thread* thread, PlaceholderTable::classlo bool PlaceholderEntry::check_seen_thread(Thread* thread, PlaceholderTable::classloadAction action) { assert_lock_strong(SystemDictionary_lock); + if (UseWispMonitor) { + thread = WispThread::current(thread); + } SeenThread* threadQ = actionToQueue(action); SeenThread* seen = threadQ; while (seen) { @@ -148,6 +152,9 @@ bool PlaceholderEntry::check_seen_thread(Thread* thread, PlaceholderTable::class // if found, deletes SeenThread bool PlaceholderEntry::remove_seen_thread(Thread* thread, PlaceholderTable::classloadAction action) { assert_lock_strong(SystemDictionary_lock); + if (UseWispMonitor) { + thread = WispThread::current(thread); + } SeenThread* threadQ = actionToQueue(action); SeenThread* seen = threadQ; SeenThread* prev = NULL; diff --git a/src/hotspot/share/classfile/protectionDomainCache.cpp b/src/hotspot/share/classfile/protectionDomainCache.cpp index b1c3e87a083..9d30aab169b 100644 --- a/src/hotspot/share/classfile/protectionDomainCache.cpp +++ b/src/hotspot/share/classfile/protectionDomainCache.cpp @@ -35,6 +35,7 @@ #include "oops/oop.inline.hpp" #include "oops/weakHandle.inline.hpp" #include "runtime/atomic.hpp" +#include "runtime/thread.hpp" #include "utilities/growableArray.hpp" #include "utilities/hashtable.inline.hpp" @@ -121,7 +122,7 @@ void ProtectionDomainCacheTable::unlink() { // point to a protection_domain that has been unloaded. // The dictionary pd_set points at entries in the ProtectionDomainCacheTable. MutexLocker ml(ClassLoaderDataGraph_lock); - MutexLocker mldict(SystemDictionary_lock); // need both. + SystemDictLocker mldict(JavaThread::current(), SystemDictionary_lock); // need both. CleanProtectionDomainEntries clean(_delete_list); ClassLoaderDataGraph::loaded_cld_do(&clean); } @@ -129,7 +130,7 @@ void ProtectionDomainCacheTable::unlink() { // Purge any deleted entries outside of the SystemDictionary_lock. purge_deleted_entries(); - MutexLocker ml(SystemDictionary_lock); + SystemDictLocker ml(JavaThread::current(), SystemDictionary_lock); int oops_removed = 0; for (int i = 0; i < table_size(); ++i) { ProtectionDomainCacheEntry** p = bucket_addr(i); diff --git a/src/hotspot/share/classfile/systemDictionary.cpp b/src/hotspot/share/classfile/systemDictionary.cpp index 03caedbbf54..cf6a44483a1 100644 --- a/src/hotspot/share/classfile/systemDictionary.cpp +++ b/src/hotspot/share/classfile/systemDictionary.cpp @@ -75,6 +75,7 @@ #include "runtime/mutexLocker.hpp" #include "runtime/sharedRuntime.hpp" #include "runtime/signature.hpp" +#include "runtime/thread.hpp" #include "services/classLoadingService.hpp" #include "services/diagnosticCommand.hpp" #include "services/threadService.hpp" @@ -223,7 +224,7 @@ Symbol* SystemDictionary::class_name_symbol(const char* name, Symbol* exception, #ifdef ASSERT // Used to verify that class loading succeeded in adding k to the dictionary. void verify_dictionary_entry(Symbol* class_name, InstanceKlass* k) { - MutexLocker mu(SystemDictionary_lock); + SystemDictLocker mu(SystemDictionary_lock); ClassLoaderData* loader_data = k->class_loader_data(); Dictionary* dictionary = loader_data->dictionary(); assert(class_name == k->name(), "Must be the same"); @@ -388,7 +389,7 @@ InstanceKlass* SystemDictionary::resolve_super_or_fail(Symbol* class_name, // can't throw error holding a lock bool throw_circularity_error = false; { - MutexLocker mu(THREAD, SystemDictionary_lock); + SystemDictLocker mu(THREAD, SystemDictionary_lock); InstanceKlass* klassk = dictionary->find_class(name_hash, class_name); InstanceKlass* quicksuperk; // To support parallel loading: if class is done loading, just return the superclass @@ -432,9 +433,9 @@ InstanceKlass* SystemDictionary::resolve_super_or_fail(Symbol* class_name, // Clean up placeholder entry. { - MutexLocker mu(THREAD, SystemDictionary_lock); + SystemDictLocker mu(THREAD, SystemDictionary_lock); placeholders()->find_and_remove(name_hash, class_name, loader_data, PlaceholderTable::LOAD_SUPER, THREAD); - SystemDictionary_lock->notify_all(); + mu.notify_all(); } // Check for pending exception or null superk, and throw exception @@ -469,7 +470,7 @@ InstanceKlass* SystemDictionary::resolve_super_or_fail(Symbol* class_name, // // The notify allows applications that did an untimed wait() on // the classloader object lock to not hang. -static void double_lock_wait(JavaThread* thread, Handle lockObject) { +static void double_lock_wait(SystemDictLocker *mu, JavaThread* thread, Handle lockObject) { assert_lock_strong(SystemDictionary_lock); assert(lockObject() != NULL, "lockObject must be non-NULL"); @@ -480,10 +481,10 @@ static void double_lock_wait(JavaThread* thread, Handle lockObject) { // These don't throw exceptions. ObjectSynchronizer::notifyall(lockObject, thread); intx recursions = ObjectSynchronizer::complete_exit(lockObject, thread); - SystemDictionary_lock->wait(); - SystemDictionary_lock->unlock(); + mu->wait(); + mu->unlock(); ObjectSynchronizer::reenter(lockObject, recursions, thread); - SystemDictionary_lock->lock(); + mu->lock(); } // If the class in is in the placeholder table, class loading is in progress. @@ -548,10 +549,11 @@ InstanceKlass* SystemDictionary::handle_parallel_loading(JavaThread* current, // which we will find below in the systemDictionary. oldprobe = NULL; // Other thread could delete this placeholder entry + SystemDictLocker mu(JavaThread::current(), SystemDictionary_lock); if (lockObject.is_null()) { - SystemDictionary_lock->wait(); + mu.wait(); } else { - double_lock_wait(current, lockObject); + double_lock_wait(&mu, current, lockObject); } // Check if classloading completed while we were waiting @@ -634,7 +636,7 @@ InstanceKlass* SystemDictionary::resolve_instance_class_or_null(Symbol* name, // Check again (after locking) if the class already exists in SystemDictionary { - MutexLocker mu(THREAD, SystemDictionary_lock); + SystemDictLocker mu(THREAD, SystemDictionary_lock); InstanceKlass* check = dictionary->find_class(name_hash, name); if (check != NULL) { // InstanceKlass is already loaded, but we still need to check protection domain below. @@ -680,7 +682,7 @@ InstanceKlass* SystemDictionary::resolve_instance_class_or_null(Symbol* name, // For these classloaders, we ensure that the first requestor // completes the load and other requestors wait for completion. { - MutexLocker mu(THREAD, SystemDictionary_lock); + SystemDictLocker mu(THREAD, SystemDictionary_lock); if (should_wait_for_loading(class_loader)) { loaded_class = handle_parallel_loading(THREAD, name_hash, @@ -728,9 +730,9 @@ InstanceKlass* SystemDictionary::resolve_instance_class_or_null(Symbol* name, // clean up placeholder entries for LOAD_INSTANCE success or error // This brackets the SystemDictionary updates for both defining // and initiating loaders - MutexLocker mu(THREAD, SystemDictionary_lock); + SystemDictLocker mu(THREAD, SystemDictionary_lock); placeholders()->find_and_remove(name_hash, name, loader_data, PlaceholderTable::LOAD_INSTANCE, THREAD); - SystemDictionary_lock->notify_all(); + SystemDictionary_lock->notify_all(THREAD); } } @@ -1486,7 +1488,7 @@ InstanceKlass* SystemDictionary::find_or_define_helper(Symbol* class_name, Handl // Hold SD lock around find_class and placeholder creation for DEFINE_CLASS { - MutexLocker mu(THREAD, SystemDictionary_lock); + SystemDictLocker mu(THREAD, SystemDictionary_lock); // First check if class already defined if (is_parallelDefine(class_loader)) { InstanceKlass* check = dictionary->find_class(name_hash, name_h); @@ -1504,7 +1506,7 @@ InstanceKlass* SystemDictionary::find_or_define_helper(Symbol* class_name, Handl // caller is surprised by LinkageError: duplicate, but findLoadedClass fails // if other thread has not finished updating dictionary while (probe->definer() != NULL) { - SystemDictionary_lock->wait(); + mu.wait(); } // Only special cases allow parallel defines and can use other thread's results // Other cases fall through, and may run into duplicate defines @@ -1512,7 +1514,7 @@ InstanceKlass* SystemDictionary::find_or_define_helper(Symbol* class_name, Handl if (is_parallelDefine(class_loader) && (probe->instance_klass() != NULL)) { InstanceKlass* ik = probe->instance_klass(); placeholders()->find_and_remove(name_hash, name_h, loader_data, PlaceholderTable::DEFINE_CLASS, THREAD); - SystemDictionary_lock->notify_all(); + SystemDictionary_lock->notify_all(THREAD); #ifdef ASSERT InstanceKlass* check = dictionary->find_class(name_hash, name_h); assert(check != NULL, "definer missed recording success"); @@ -1528,7 +1530,7 @@ InstanceKlass* SystemDictionary::find_or_define_helper(Symbol* class_name, Handl // definer must notify any waiting threads { - MutexLocker mu(THREAD, SystemDictionary_lock); + SystemDictLocker mu(THREAD, SystemDictionary_lock); PlaceholderEntry* probe = placeholders()->get_entry(name_hash, name_h, loader_data); assert(probe != NULL, "DEFINE_CLASS placeholder lost?"); if (!HAS_PENDING_EXCEPTION) { @@ -1536,7 +1538,7 @@ InstanceKlass* SystemDictionary::find_or_define_helper(Symbol* class_name, Handl } probe->set_definer(NULL); placeholders()->find_and_remove(name_hash, name_h, loader_data, PlaceholderTable::DEFINE_CLASS, THREAD); - SystemDictionary_lock->notify_all(); + SystemDictionary_lock->notify_all(THREAD); } return HAS_PENDING_EXCEPTION ? NULL : k; @@ -1604,7 +1606,7 @@ bool SystemDictionary::do_unloading(GCTimer* gc_timer) { MutexLocker ml2(is_concurrent ? Module_lock : NULL); JFR_ONLY(Jfr::on_unloading_classes();) - MutexLocker ml1(is_concurrent ? SystemDictionary_lock : NULL); + SystemDictLocker ml1(JavaThread::current(), is_concurrent ? SystemDictionary_lock : NULL); ClassLoaderDataGraph::clean_module_and_package_info(); constraints()->purge_loader_constraints(); resolution_errors()->purge_resolution_errors(); @@ -1681,7 +1683,7 @@ void SystemDictionary::check_constraints(unsigned int name_hash, Symbol *name = k->name(); ClassLoaderData *loader_data = class_loader_data(class_loader); - MutexLocker mu(THREAD, SystemDictionary_lock); + SystemDictLocker mu(THREAD, SystemDictionary_lock); InstanceKlass* check = loader_data->dictionary()->find_class(name_hash, name); if (check != NULL) { @@ -1736,7 +1738,7 @@ void SystemDictionary::update_dictionary(unsigned int hash, ClassLoaderData *loader_data = class_loader_data(class_loader); { - MutexLocker mu1(SystemDictionary_lock); + SystemDictLocker mu1(JavaThread::current(), SystemDictionary_lock); // Make a new dictionary entry. Dictionary* dictionary = loader_data->dictionary(); @@ -1744,7 +1746,7 @@ void SystemDictionary::update_dictionary(unsigned int hash, if (sd_check == NULL) { dictionary->add_klass(hash, name, k); } - SystemDictionary_lock->notify_all(); + mu1.notify_all(); } } @@ -1775,7 +1777,7 @@ Klass* SystemDictionary::find_constrained_instance_or_array_klass( if (t != T_OBJECT) { klass = Universe::typeArrayKlassObj(t); } else { - MutexLocker mu(current, SystemDictionary_lock); + SystemDictLocker mu(JavaThread::current(), SystemDictionary_lock); klass = constraints()->find_constrained_klass(ss.as_symbol(), class_loader); } // If element class already loaded, allocate array klass @@ -1783,7 +1785,7 @@ Klass* SystemDictionary::find_constrained_instance_or_array_klass( klass = klass->array_klass_or_null(ndims); } } else { - MutexLocker mu(current, SystemDictionary_lock); + SystemDictLocker mu(JavaThread::current(), SystemDictionary_lock); // Non-array classes are easy: simply check the constraint table. klass = constraints()->find_constrained_klass(class_name, class_loader); } @@ -1824,7 +1826,7 @@ bool SystemDictionary::add_loader_constraint(Symbol* class_name, unsigned int name_hash2 = dictionary2->compute_hash(constraint_name); { - MutexLocker mu_s(SystemDictionary_lock); + SystemDictLocker mu_s(JavaThread::current(), SystemDictionary_lock); InstanceKlass* klass1 = dictionary1->find_class(name_hash1, constraint_name); InstanceKlass* klass2 = dictionary2->find_class(name_hash2, constraint_name); bool result = constraints()->add_entry(constraint_name, klass1, class_loader1, @@ -1852,7 +1854,7 @@ void SystemDictionary::add_resolution_error(const constantPoolHandle& pool, int unsigned int hash = resolution_errors()->compute_hash(pool, which); int index = resolution_errors()->hash_to_index(hash); { - MutexLocker ml(Thread::current(), SystemDictionary_lock); + SystemDictLocker ml(JavaThread::current(), SystemDictionary_lock); ResolutionErrorEntry* entry = resolution_errors()->find_entry(index, hash, pool, which); if (entry == NULL) { resolution_errors()->add_entry(index, hash, pool, which, error, message, cause, cause_msg); @@ -1871,7 +1873,7 @@ Symbol* SystemDictionary::find_resolution_error(const constantPoolHandle& pool, unsigned int hash = resolution_errors()->compute_hash(pool, which); int index = resolution_errors()->hash_to_index(hash); { - MutexLocker ml(Thread::current(), SystemDictionary_lock); + SystemDictLocker ml(JavaThread::current(), SystemDictionary_lock); ResolutionErrorEntry* entry = resolution_errors()->find_entry(index, hash, pool, which); if (entry != NULL) { *message = entry->message(); @@ -1894,7 +1896,7 @@ void SystemDictionary::add_nest_host_error(const constantPoolHandle& pool, unsigned int hash = resolution_errors()->compute_hash(pool, which); int index = resolution_errors()->hash_to_index(hash); { - MutexLocker ml(Thread::current(), SystemDictionary_lock); + SystemDictLocker ml(JavaThread::current(), SystemDictionary_lock); ResolutionErrorEntry* entry = resolution_errors()->find_entry(index, hash, pool, which); if (entry != NULL && entry->nest_host_error() == NULL) { // An existing entry means we had a true resolution failure (LinkageError) with our nest host, but we @@ -1913,7 +1915,7 @@ const char* SystemDictionary::find_nest_host_error(const constantPoolHandle& poo unsigned int hash = resolution_errors()->compute_hash(pool, which); int index = resolution_errors()->hash_to_index(hash); { - MutexLocker ml(Thread::current(), SystemDictionary_lock); + SystemDictLocker ml(JavaThread::current(), SystemDictionary_lock); ResolutionErrorEntry* entry = resolution_errors()->find_entry(index, hash, pool, which); if (entry != NULL) { return entry->nest_host_error(); @@ -2028,7 +2030,7 @@ Method* SystemDictionary::find_method_handle_intrinsic(vmIntrinsicID iid, // Now grab the lock. We might have to throw away the new method, // if a racing thread has managed to install one at the same time. { - MutexLocker ml(THREAD, SystemDictionary_lock); + SystemDictLocker ml(THREAD, SystemDictionary_lock); spe = invoke_method_table()->find_entry(index, hash, signature, iid_as_int); if (spe == NULL) spe = invoke_method_table()->add_entry(index, hash, signature, iid_as_int); @@ -2250,7 +2252,7 @@ Handle SystemDictionary::find_method_handle_type(Symbol* signature, if (can_be_cached) { // We can cache this MethodType inside the JVM. - MutexLocker ml(THREAD, SystemDictionary_lock); + SystemDictLocker ml(THREAD, SystemDictionary_lock); spe = invoke_method_table()->find_entry(index, hash, signature, null_iid); if (spe == NULL) spe = invoke_method_table()->add_entry(index, hash, signature, null_iid); @@ -2407,7 +2409,7 @@ bool SystemDictionary::is_nonpublic_Object_method(Method* m) { void SystemDictionary::print_on(outputStream *st) { CDS_ONLY(SystemDictionaryShared::print_on(st)); - GCMutexLocker mu(SystemDictionary_lock); + GCSystemDictLocker mu(SystemDictionary_lock); ClassLoaderDataGraph::print_dictionary(st); @@ -2431,7 +2433,7 @@ void SystemDictionary::verify() { guarantee(placeholders()->number_of_entries() >= 0, "Verify of placeholders failed"); - GCMutexLocker mu(SystemDictionary_lock); + GCSystemDictLocker mu(SystemDictionary_lock); // Verify dictionary ClassLoaderDataGraph::verify_dictionary(); @@ -2459,17 +2461,17 @@ void SystemDictionary::dump(outputStream *st, bool verbose) { } TableStatistics SystemDictionary::placeholders_statistics() { - MutexLocker ml(SystemDictionary_lock); + SystemDictLocker ml(JavaThread::current(), SystemDictionary_lock); return placeholders()->statistics_calculate(); } TableStatistics SystemDictionary::loader_constraints_statistics() { - MutexLocker ml(SystemDictionary_lock); + SystemDictLocker ml(JavaThread::current(), SystemDictionary_lock); return constraints()->statistics_calculate(); } TableStatistics SystemDictionary::protection_domain_cache_statistics() { - MutexLocker ml(SystemDictionary_lock); + SystemDictLocker ml(JavaThread::current(), SystemDictionary_lock); return pd_cache_table()->statistics_calculate(); } @@ -2486,3 +2488,8 @@ void SystemDictionaryDCmd::execute(DCmdSource source, TRAPS) { _verbose.value()); VMThread::execute(&dumper); } + +void SystemDictionary::system_dict_lock_change(JavaThread* THREAD) { + assert(UseWispMonitor, "UseWispMonitor is off"); + SystemDictionary_lock->set_obj_lock(oopFactory::new_intArray(0, THREAD), THREAD); +} diff --git a/src/hotspot/share/classfile/systemDictionary.hpp b/src/hotspot/share/classfile/systemDictionary.hpp index f4d40685ddc..11dd5dc081a 100644 --- a/src/hotspot/share/classfile/systemDictionary.hpp +++ b/src/hotspot/share/classfile/systemDictionary.hpp @@ -416,6 +416,9 @@ class SystemDictionary : AllStatic { static TableStatistics placeholders_statistics(); static TableStatistics loader_constraints_statistics(); static TableStatistics protection_domain_cache_statistics(); + +public: + static void system_dict_lock_change(TRAPS); }; #endif // SHARE_CLASSFILE_SYSTEMDICTIONARY_HPP diff --git a/src/hotspot/share/classfile/systemDictionaryShared.cpp b/src/hotspot/share/classfile/systemDictionaryShared.cpp index a055ad74156..dfba773d38c 100644 --- a/src/hotspot/share/classfile/systemDictionaryShared.cpp +++ b/src/hotspot/share/classfile/systemDictionaryShared.cpp @@ -1009,7 +1009,7 @@ InstanceKlass* SystemDictionaryShared::find_or_load_shared_class( // which are parallel-capable loaders, so a lock here is NOT taken. assert(get_loader_lock_or_null(class_loader) == NULL, "ObjectLocker not required"); { - MutexLocker mu(THREAD, SystemDictionary_lock); + SystemDictLocker mu(THREAD, SystemDictionary_lock); InstanceKlass* check = dictionary->find_class(d_hash, name); if (check != NULL) { return check; diff --git a/src/hotspot/share/classfile/vmClassMacros.hpp b/src/hotspot/share/classfile/vmClassMacros.hpp index 53bb2a41f74..665fe24bf23 100644 --- a/src/hotspot/share/classfile/vmClassMacros.hpp +++ b/src/hotspot/share/classfile/vmClassMacros.hpp @@ -171,6 +171,8 @@ /* Stack manipulation classes */ \ do_klass(java_dyn_CoroutineSupport_klass, java_dyn_CoroutineSupport ) \ do_klass(java_dyn_CoroutineBase_klass, java_dyn_CoroutineBase ) \ + do_klass(com_alibaba_wisp_engine_WispTask_klass, com_alibaba_wisp_engine_WispTask ) \ + do_klass(com_alibaba_wisp_engine_WispEngine_klass, com_alibaba_wisp_engine_WispEngine ) \ \ /* force inline of iterators */ \ do_klass(Iterator_klass, java_util_Iterator ) \ diff --git a/src/hotspot/share/classfile/vmIntrinsics.hpp b/src/hotspot/share/classfile/vmIntrinsics.hpp index 02488180b22..2f4cd62f16a 100644 --- a/src/hotspot/share/classfile/vmIntrinsics.hpp +++ b/src/hotspot/share/classfile/vmIntrinsics.hpp @@ -233,7 +233,7 @@ class methodHandle; do_name( arraycopy_name, "arraycopy") \ do_signature(arraycopy_signature, "(Ljava/lang/Object;ILjava/lang/Object;II)V") \ do_intrinsic(_currentThread, java_lang_Thread, currentThread_name, currentThread_signature, F_S) \ - do_name( currentThread_name, "currentThread") \ + do_name( currentThread_name, "currentThread0") \ do_signature(currentThread_signature, "()Ljava/lang/Thread;") \ \ /* reflective intrinsics, for java/lang/Class, etc. */ \ @@ -935,11 +935,12 @@ class methodHandle; \ /* (2) Bytecode intrinsics */ \ \ - do_intrinsic(_park, jdk_internal_misc_Unsafe, park_name, park_signature, F_R) \ + do_intrinsic(_park, jdk_internal_misc_Unsafe, park0_name, park_signature, F_R) \ do_name( park_name, "park") \ + do_name( park0_name, "park0") \ do_signature(park_signature, "(ZJ)V") \ - do_intrinsic(_unpark, jdk_internal_misc_Unsafe, unpark_name, unpark_signature, F_R) \ - do_name( unpark_name, "unpark") \ + do_intrinsic(_unpark, jdk_internal_misc_Unsafe, unpark0_name, unpark_signature, F_R) \ + do_name( unpark0_name, "unpark0") \ do_alias( unpark_signature, /*(LObject;)V*/ object_void_signature) \ \ /* coroutine intrinsics */ \ diff --git a/src/hotspot/share/classfile/vmSymbols.hpp b/src/hotspot/share/classfile/vmSymbols.hpp index 03d49dddc1c..8e7adcf2cf2 100644 --- a/src/hotspot/share/classfile/vmSymbols.hpp +++ b/src/hotspot/share/classfile/vmSymbols.hpp @@ -477,6 +477,8 @@ template(module_entry_name, "module_entry") \ template(resolved_references_name, "") \ template(init_lock_name, "") \ + template(initializeWispClass_name, "initializeWispClass") \ + template(startWispDaemons_name, "startWispDaemons") \ template(address_size_name, "ADDRESS_SIZE0") \ template(page_size_name, "PAGE_SIZE") \ template(big_endian_name, "BIG_ENDIAN") \ @@ -688,18 +690,44 @@ template(java_dyn_CoroutineSupport, "java/dyn/CoroutineSupport") \ template(java_dyn_CoroutineBase, "java/dyn/CoroutineBase") \ template(java_dyn_CoroutineExitException, "java/dyn/CoroutineExitException") \ + template(com_alibaba_wisp_engine_WispTask, "com/alibaba/wisp/engine/WispTask") \ + template(com_alibaba_wisp_engine_WispEngine, "com/alibaba/wisp/engine/WispEngine") \ + template(isInCritical_name, "isInCritical") \ + template(jvmParkStatus_name, "jvmParkStatus") \ + template(id_name, "id") \ + template(threadWrapper_name, "threadWrapper") \ + template(activeCount_name, "activeCount") \ + template(stealCount_name, "stealCount") \ + template(stealFailureCount_name, "stealFailureCount") \ + template(unparkById_name, "unparkById") \ + template(interruptById_name, "interruptById") \ + template(interrupted_name, "interrupted") \ + template(yield_name, "yield") \ template(data_name, "data") \ template(stack_name, "stack") \ template(current_name, "current") \ template(java_dyn_CoroutineBase_signature, "Ljava/dyn/CoroutineBase;") \ template(startInternal_method_name, "startInternal") \ template(initializeCoroutineSupport_method_name, "initializeCoroutineSupport") \ + template(destroyCoroutineSupport_method_name, "destroyCoroutineSupport") \ template(bci_name, "bci") \ template(localCount_name, "localCount") \ template(expressionCount_name, "expressionCount") \ template(scalarValues_name, "scalarValues") \ template(objectValues_name, "objectValues") \ \ + /* coroutine work steal support */ \ + template(doPrivileged_name, "doPrivileged") \ + template(doPrivileged_signature_1, "(Ljava/security/PrivilegedAction;)Ljava/lang/Object;") \ + template(doPrivileged_signature_2, "(Ljava/security/PrivilegedAction;Ljava/security/AccessControlContext;)Ljava/lang/Object;") \ + template(doPrivileged_signature_3, "(Ljava/security/PrivilegedExceptionAction;)Ljava/lang/Object;") \ + template(doPrivileged_signature_4, "(Ljava/security/PrivilegedExceptionAction;Ljava/security/AccessControlContext;)Ljava/lang/Object;") \ + template(sun_reflect_NativeMethodAccessorImpl, "sun/reflect/NativeMethodAccessorImpl") \ + template(invoke0_name, "invoke0") \ + template(invoke0_signature, "(Ljava/lang/reflect/Method;Ljava/lang/Object;[Ljava/lang/Object;)Ljava/lang/Object;") \ + template(sun_reflect_NativeConstructorAccessorImpl, "sun/reflect/NativeConstructorAccessorImpl") \ + template(newInstance0_signature, "(Ljava/lang/reflect/Constructor;[Ljava/lang/Object;)Ljava/lang/Object;") \ + \ /* forEachRemaining support */ \ template(java_util_stream_StreamsRangeIntSpliterator, "java/util/stream/Streams$RangeIntSpliterator") \ \ diff --git a/src/hotspot/share/include/jvm.h b/src/hotspot/share/include/jvm.h index f12a4bec3d0..5232c53c653 100644 --- a/src/hotspot/share/include/jvm.h +++ b/src/hotspot/share/include/jvm.h @@ -257,6 +257,9 @@ JVM_StartThread(JNIEnv *env, jobject thread); JNIEXPORT void JNICALL JVM_StopThread(JNIEnv *env, jobject thread, jobject exception); +JNIEXPORT jboolean JNICALL +JVM_CheckAndClearNativeInterruptForWisp(JNIEnv* env, jobject task, jobject thread); + JNIEXPORT jboolean JNICALL JVM_IsThreadAlive(JNIEnv *env, jobject thread); @@ -293,6 +296,9 @@ JVM_GetAllThreads(JNIEnv *env, jclass dummy); JNIEXPORT void JNICALL JVM_SetNativeThreadName(JNIEnv *env, jobject jthread, jstring name); +JNIEXPORT jboolean JNICALL +JVM_IsInNative(JNIEnv *env, jobject jthread); + /* getStackTrace() and getAllStackTraces() method */ JNIEXPORT jobjectArray JNICALL JVM_DumpThreads(JNIEnv *env, jclass threadClass, jobjectArray threads); @@ -1043,6 +1049,14 @@ JVM_ReleaseUTF(const char *utf); JNIEXPORT jboolean JNICALL JVM_IsSameClassPackage(JNIEnv *env, jclass class1, jclass class2); +JNIEXPORT void JNICALL +JVM_SetWispTask(JNIEnv* env, jclass clz, jlong coroutinePtr, jint task_id, jobject task, jobject engine); + +JNIEXPORT jint JNICALL +JVM_GetProxyUnpark(JNIEnv* env, jclass clz, jintArray res); + +JNIEXPORT void JNICALL +JVM_MarkPreempt(JNIEnv* env, jclass clz, jobject thread); /************************************************************************* PART 3: I/O and Network Support diff --git a/src/hotspot/share/interpreter/interpreterRuntime.cpp b/src/hotspot/share/interpreter/interpreterRuntime.cpp index d66ed24d862..e9d68d12b77 100644 --- a/src/hotspot/share/interpreter/interpreterRuntime.cpp +++ b/src/hotspot/share/interpreter/interpreterRuntime.cpp @@ -722,6 +722,8 @@ void InterpreterRuntime::resolve_get_put(JavaThread* current, Bytecodes::Code by // be shared by method invocation and synchronized blocks. //%note synchronization_3 +address monitorenter_address_interp = (address)InterpreterRuntime::monitorenter; + //%note monitor_1 JRT_ENTRY_NO_ASYNC(void, InterpreterRuntime::monitorenter(JavaThread* current, BasicObjectLock* elem)) #ifdef ASSERT @@ -730,6 +732,14 @@ JRT_ENTRY_NO_ASYNC(void, InterpreterRuntime::monitorenter(JavaThread* current, B if (PrintBiasedLockingStatistics) { Atomic::inc(BiasedLocking::slow_path_entry_count_addr()); } + + Thread* t = THREAD; + // thread steal support + WispPostStealHandleUpdateMark w(current, t, __tiv, __hm); + + // Coroutine work steal support + EnableStealMark p(THREAD); + Handle h_obj(current, elem->obj()); assert(Universe::heap()->is_in_or_null(h_obj()), "must be NULL or an object"); @@ -772,15 +782,24 @@ JRT_ENTRY(void, InterpreterRuntime::new_illegal_monitor_state_exception(JavaThre // method will be called during an exception unwind. assert(!HAS_PENDING_EXCEPTION, "no pending exception"); - Handle exception(current, current->vm_result()); + // this path will also use vm_result, so we'd hook it again. + Handle exception(current, (EnableCoroutine && UseWispMonitor) ? current->vm_result_for_wisp() : current->vm_result()); assert(exception() != NULL, "vm result should be set"); - current->set_vm_result(NULL); // clear vm result before continuing (may cause memory leaks and assert failures) + if (EnableCoroutine && UseWispMonitor) { + current->set_vm_result_for_wisp(NULL); + } else { + current->set_vm_result(NULL); // clear vm result before continuing (may cause memory leaks and assert failures) + } if (!exception->is_a(vmClasses::ThreadDeath_klass())) { exception = get_preinitialized_exception( vmClasses::IllegalMonitorStateException_klass(), CATCH); } - current->set_vm_result(exception()); + if (EnableCoroutine && UseWispMonitor) { + current->set_vm_result_for_wisp(exception()); + } else { + current->set_vm_result(exception()); + } JRT_END diff --git a/src/hotspot/share/jvmci/jvmciRuntime.cpp b/src/hotspot/share/jvmci/jvmciRuntime.cpp index 29ff5985b37..aca748d2442 100644 --- a/src/hotspot/share/jvmci/jvmciRuntime.cpp +++ b/src/hotspot/share/jvmci/jvmciRuntime.cpp @@ -392,6 +392,7 @@ address JVMCIRuntime::exception_handler_for_pc(JavaThread* current) { } JRT_BLOCK_ENTRY(void, JVMCIRuntime::monitorenter(JavaThread* current, oopDesc* obj, BasicLock* lock)) +WispPostStealHandleUpdateMark w(__hm); SharedRuntime::monitor_enter_helper(obj, lock, current); JRT_END diff --git a/src/hotspot/share/memory/allocation.hpp b/src/hotspot/share/memory/allocation.hpp index e881b577b35..3d78aa19fff 100644 --- a/src/hotspot/share/memory/allocation.hpp +++ b/src/hotspot/share/memory/allocation.hpp @@ -141,6 +141,7 @@ class AllocatedObj { f(mtArguments, "Arguments") \ f(mtModule, "Module") \ f(mtSafepoint, "Safepoint") \ + f(mtWisp, "Wisp") /* memory used by Wisp cod */ \ f(mtSynchronizer, "Synchronization") \ f(mtServiceability, "Serviceability") \ f(mtMetaspace, "Metaspace") \ diff --git a/src/hotspot/share/memory/arena.hpp b/src/hotspot/share/memory/arena.hpp index 1b38046a98a..b61352852af 100644 --- a/src/hotspot/share/memory/arena.hpp +++ b/src/hotspot/share/memory/arena.hpp @@ -87,6 +87,8 @@ class Chunk: CHeapObj { static void start_chunk_pool_cleaner_task(); }; +class WispPostStealHandleUpdateMark; + //------------------------------Arena------------------------------------------ // Fast allocation of memory class Arena : public CHeapObj { @@ -94,6 +96,7 @@ class Arena : public CHeapObj { friend class HandleMark; friend class NoHandleMark; friend class VMStructs; + friend class WispPostStealHandleUpdateMark; MEMFLAGS _flags; // Memory tracking flags diff --git a/src/hotspot/share/memory/resourceArea.hpp b/src/hotspot/share/memory/resourceArea.hpp index ff3effe7589..34336c942a3 100644 --- a/src/hotspot/share/memory/resourceArea.hpp +++ b/src/hotspot/share/memory/resourceArea.hpp @@ -39,10 +39,13 @@ // } // ... +class WispPostStealHandleUpdateMark; + //------------------------------ResourceArea----------------------------------- // A ResourceArea is an Arena that supports safe usage of ResourceMark. class ResourceArea: public Arena { friend class VMStructs; + friend class WispPostStealHandleUpdateMark; #ifdef ASSERT int _nesting; // current # of nested ResourceMarks diff --git a/src/hotspot/share/oops/instanceKlass.cpp b/src/hotspot/share/oops/instanceKlass.cpp index bf9416915ce..943e7269768 100644 --- a/src/hotspot/share/oops/instanceKlass.cpp +++ b/src/hotspot/share/oops/instanceKlass.cpp @@ -77,6 +77,7 @@ #include "runtime/atomic.hpp" #include "runtime/biasedLocking.hpp" #include "runtime/fieldDescriptor.inline.hpp" +#include "runtime/coroutine.hpp" #include "runtime/handles.inline.hpp" #include "runtime/javaCalls.hpp" #include "runtime/mutexLocker.hpp" @@ -4187,6 +4188,23 @@ unsigned char * InstanceKlass::get_cached_class_file_bytes() { } #endif +bool InstanceKlass::is_reentrant_initialization(Thread *thread) { + if (UseWispMonitor) { + assert(thread != NULL, "sanity check"); + thread = WispThread::current(thread); + } + return thread == _init_thread; +} + +void InstanceKlass::set_init_thread(Thread *thread) { + if (UseWispMonitor && thread != NULL) { + assert(thread->is_Java_thread(), "sanity check"); + assert(((JavaThread*) thread)->current_coroutine() != NULL, "sanity check"); + thread = WispThread::current(thread); + } + _init_thread = thread; +} + bool InstanceKlass::is_shareable() const { #if INCLUDE_CDS ClassLoaderData* loader_data = class_loader_data(); diff --git a/src/hotspot/share/oops/instanceKlass.hpp b/src/hotspot/share/oops/instanceKlass.hpp index 211788d6925..22dbff26877 100644 --- a/src/hotspot/share/oops/instanceKlass.hpp +++ b/src/hotspot/share/oops/instanceKlass.hpp @@ -546,7 +546,7 @@ class InstanceKlass: public Klass { bool is_not_initialized() const { return _init_state < being_initialized; } bool is_being_initialized() const { return _init_state == being_initialized; } bool is_in_error_state() const { return _init_state == initialization_error; } - bool is_reentrant_initialization(Thread *thread) { return thread == _init_thread; } + bool is_reentrant_initialization(Thread *thread); ClassState init_state() { return (ClassState)_init_state; } bool is_rewritten() const { return (_misc_flags & _misc_rewritten) != 0; } @@ -1170,7 +1170,7 @@ class InstanceKlass: public Klass { // initialization state void set_init_state(ClassState state); void set_rewritten() { _misc_flags |= _misc_rewritten; } - void set_init_thread(Thread *thread) { _init_thread = thread; } + void set_init_thread(Thread *thread); // The RedefineClasses() API can cause new method idnums to be needed // which will cause the caches to grow. Safety requires different diff --git a/src/hotspot/share/opto/classes.hpp b/src/hotspot/share/opto/classes.hpp index 582ec51fbde..37edfe88e3c 100644 --- a/src/hotspot/share/opto/classes.hpp +++ b/src/hotspot/share/opto/classes.hpp @@ -337,6 +337,7 @@ macro(TailCall) macro(TailJump) macro(MacroLogicV) macro(ThreadLocal) +macro(ThreadRefetch) macro(Unlock) macro(URShiftB) macro(URShiftS) diff --git a/src/hotspot/share/opto/connode.hpp b/src/hotspot/share/opto/connode.hpp index ef0318b2275..d58bb088dd4 100644 --- a/src/hotspot/share/opto/connode.hpp +++ b/src/hotspot/share/opto/connode.hpp @@ -149,6 +149,21 @@ class ThreadLocalNode : public Node { virtual uint ideal_reg() const { return Op_RegP; } }; +// We use this node in gen_stub() while generating monitorenter stub. +// Our monitorenter stub will call java for `park`, when it is blocked, +// it may be stolen by another thread; but we cannot fix the r15 inside the +// `SharedRuntime::complete_monitor_locking_C` because it obeys the x86 +// calling convention. So we should update it after returning on this function.. +// This node is used to: when `SharedRuntime::complete_monitor_locking_C` returns, we +// immediately change r15 by using this node. About its MachNode definition, +// please check `tlsRefetchP` in x86_64.ad for more detail. +class ThreadRefetchNode : public Node { +public: + ThreadRefetchNode(Node *control, Node *load) : Node(control, load) {} + virtual int Opcode() const; + virtual const Type *bottom_type() const { return TypeRawPtr::BOTTOM;} + virtual uint ideal_reg() const { return Op_RegP; } +}; #endif // SHARE_OPTO_CONNODE_HPP diff --git a/src/hotspot/share/opto/generateOptoStub.cpp b/src/hotspot/share/opto/generateOptoStub.cpp index c5dfd1eeb40..a1e65aeb3b6 100644 --- a/src/hotspot/share/opto/generateOptoStub.cpp +++ b/src/hotspot/share/opto/generateOptoStub.cpp @@ -28,6 +28,7 @@ #include "opto/callnode.hpp" #include "opto/cfgnode.hpp" #include "opto/compile.hpp" +#include "opto/connode.hpp" #include "opto/convertnode.hpp" #include "opto/locknode.hpp" #include "opto/memnode.hpp" @@ -103,6 +104,12 @@ void GraphKit::gen_stub(address C_function, Node *last_sp = frameptr(); store_to_memory(control(), adr_sp, last_sp, T_ADDRESS, NoAlias, MemNode::unordered); + Node *coro_task = NULL; + if (EnableSteal && C_function == (address)SharedRuntime::complete_monitor_locking_C) { + Node *coro_adr = basic_plus_adr(top(), thread, in_bytes(JavaThread::current_coroutine_offset())); + coro_task = make_load(NULL, coro_adr, TypeRawPtr::NOTNULL, T_ADDRESS, MemNode::unordered); + } + // Set _thread_in_native // The order of stores into TLS is critical! Setting _thread_in_native MUST // be last, because a GC is allowed at any time after setting it and the GC @@ -220,6 +227,19 @@ void GraphKit::gen_stub(address C_function, //----------------------------- + Node *refetch = NULL; + if (EnableSteal && C_function == (address)SharedRuntime::complete_monitor_locking_C) { + // we add ThreadRefetchNode so we can refetch r15 when monitorenter() ends, and + // after refetching thread, every node based on ThreadLocalNode should be changed to use ThreadRefetchNode as thread. + refetch = _gvn.transform(new ThreadRefetchNode(control(), coro_task)); + adr_sp = basic_plus_adr(top(), refetch, in_bytes(JavaThread::last_Java_sp_offset())); + adr_last_Java_pc = basic_plus_adr(top(), + refetch, + in_bytes(JavaThread::frame_anchor_offset()) + + in_bytes(JavaFrameAnchor::last_Java_pc_offset())); + + } + // Clear last_Java_sp store_to_memory(control(), adr_sp, null(), T_ADDRESS, NoAlias, MemNode::unordered); // Clear last_Java_pc @@ -242,7 +262,7 @@ void GraphKit::gen_stub(address C_function, //----------------------------- // check exception - Node* adr = basic_plus_adr(top(), thread, in_bytes(Thread::pending_exception_offset())); + Node* adr = basic_plus_adr(top(), !refetch ? thread : refetch, in_bytes(Thread::pending_exception_offset())); Node* pending = make_load(NULL, adr, TypeOopPtr::BOTTOM, T_OBJECT, NoAlias, MemNode::unordered); Node* exit_memory = reset_memory(); diff --git a/src/hotspot/share/opto/graphKit.cpp b/src/hotspot/share/opto/graphKit.cpp index a3df43c2381..b50e7abb870 100644 --- a/src/hotspot/share/opto/graphKit.cpp +++ b/src/hotspot/share/opto/graphKit.cpp @@ -48,6 +48,7 @@ #include "opto/subtypenode.hpp" #include "runtime/deoptimization.hpp" #include "runtime/sharedRuntime.hpp" +#include "runtime/coroutine.hpp" #include "utilities/bitMap.inline.hpp" #include "utilities/powerOfTwo.hpp" #include "utilities/growableArray.hpp" @@ -850,6 +851,148 @@ static bool should_reexecute_implied_by_bytecode(JVMState *jvms, bool is_anewarr } } +void GraphKit::add_safepoint_edges_wisp(SafePointNode* call) { + assert(UseWispMonitor, "UseWispMonitor is off"); + // Add the safepoint edges to the call (or other safepoint). + + // Walk the inline list to fill in the correct set of JVMState's + // Also fill in the associated edges for each JVMState. + + // If the bytecode needs to be reexecuted we need to put + // the arguments back on the stack. + const bool should_reexecute = jvms()->should_reexecute(); + JVMState* youngest_jvms = should_reexecute ? sync_jvms_for_reexecute() : sync_jvms(); + + // NOTE: set_bci (called from sync_jvms) might reset the reexecute bit to + // undefined if the bci is different. This is normal for Parse but it + // should not happen for LibraryCallKit because only one bci is processed. + assert(!is_LibraryCallKit() || (jvms()->should_reexecute() == should_reexecute), + "in LibraryCallKit the reexecute bit should not change"); + + // If we are guaranteed to throw, we can prune everything but the + // input to the current bytecode. + bool can_prune_locals = false; + uint stack_slots_not_pruned = 0; + int inputs = 0, depth = 0; + + if (env()->should_retain_local_variables()) { + // At any safepoint, this method can get breakpointed, which would + // then require an immediate deoptimization. + can_prune_locals = false; // do not prune locals + stack_slots_not_pruned = 0; + } + + // do not scribble on the input jvms + JVMState* out_jvms = youngest_jvms->clone_deep(C); + call->set_jvms(out_jvms); // Start jvms list for call node + + // Presize the call: + DEBUG_ONLY(uint non_debug_edges = call->req()); + call->add_req_batch(top(), youngest_jvms->debug_depth()); + assert(call->req() == non_debug_edges + youngest_jvms->debug_depth(), ""); + + // Set up edges so that the call looks like this: + // Call [state:] ctl io mem fptr retadr + // [parms:] parm0 ... parmN + // [root:] loc0 ... locN stk0 ... stkSP mon0 obj0 ... monN objN + // [...mid:] loc0 ... locN stk0 ... stkSP mon0 obj0 ... monN objN [...] + // [young:] loc0 ... locN stk0 ... stkSP mon0 obj0 ... monN objN + // Note that caller debug info precedes callee debug info. + + // Fill pointer walks backwards from "young:" to "root:" in the diagram above: + uint debug_ptr = call->req(); + + // Loop over the map input edges associated with jvms, add them + // to the call node, & reset all offsets to match call node array. + for (JVMState* in_jvms = youngest_jvms; in_jvms != NULL; ) { + uint debug_end = debug_ptr; + uint debug_start = debug_ptr - in_jvms->debug_size(); + debug_ptr = debug_start; // back up the ptr + + uint p = debug_start; // walks forward in [debug_start, debug_end) + uint j, k, l; + SafePointNode* in_map = in_jvms->map(); + out_jvms->set_map(call); + + if (can_prune_locals) { + assert(in_jvms->method() == out_jvms->method(), "sanity"); + // If the current throw can reach an exception handler in this JVMS, + // then we must keep everything live that can reach that handler. + // As a quick and dirty approximation, we look for any handlers at all. + if (in_jvms->method()->has_exception_handlers()) { + can_prune_locals = false; + } + } + + // Add the Locals + k = in_jvms->locoff(); + l = in_jvms->loc_size(); + out_jvms->set_locoff(p); + if (!can_prune_locals) { + for (j = 0; j < l; j++) + call->set_req(p++, in_map->in(k+j)); + } else { + p += l; // already set to top above by add_req_batch + } + + // Add the Expression Stack + k = in_jvms->stkoff(); + l = in_jvms->sp(); + out_jvms->set_stkoff(p); + if (!can_prune_locals) { + for (j = 0; j < l; j++) + call->set_req(p++, in_map->in(k+j)); + } else if (can_prune_locals && stack_slots_not_pruned != 0) { + // Divide stack into {S0,...,S1}, where S0 is set to top. + uint s1 = stack_slots_not_pruned; + stack_slots_not_pruned = 0; // for next iteration + if (s1 > l) s1 = l; + uint s0 = l - s1; + p += s0; // skip the tops preinstalled by add_req_batch + for (j = s0; j < l; j++) + call->set_req(p++, in_map->in(k+j)); + } else { + p += l; // already set to top above by add_req_batch + } + + // Add the Monitors + k = in_jvms->monoff(); + l = in_jvms->mon_size(); + out_jvms->set_monoff(p); + for (j = 0; j < l; j++) + call->set_req(p++, in_map->in(k+j)); + + // Copy any scalar object fields. + k = in_jvms->scloff(); + l = in_jvms->scl_size(); + out_jvms->set_scloff(p); + for (j = 0; j < l; j++) + call->set_req(p++, in_map->in(k+j)); + + // Finish the new jvms. + out_jvms->set_endoff(p); + + assert(out_jvms->endoff() == debug_end, "fill ptr must match"); + assert(out_jvms->depth() == in_jvms->depth(), "depth must match"); + assert(out_jvms->loc_size() == in_jvms->loc_size(), "size must match"); + assert(out_jvms->mon_size() == in_jvms->mon_size(), "size must match"); + assert(out_jvms->scl_size() == in_jvms->scl_size(), "size must match"); + assert(out_jvms->debug_size() == in_jvms->debug_size(), "size must match"); + + // Update the two tail pointers in parallel. + out_jvms = out_jvms->caller(); + in_jvms = in_jvms->caller(); + } + + assert(debug_ptr == non_debug_edges, "debug info must fit exactly"); + + // Test the correctness of JVMState::debug_xxx accessors: + assert(call->jvms()->debug_start() == non_debug_edges, ""); + assert(call->jvms()->debug_end() == call->req(), ""); + assert(call->jvms()->debug_depth() == call->req() - non_debug_edges, ""); +} + + // Helper function for adding JVMState and debug information to node void GraphKit::add_safepoint_edges(SafePointNode* call, bool must_throw) { // Add the safepoint edges to the call (or other safepoint). @@ -2482,7 +2625,8 @@ Node* GraphKit::make_runtime_call(int flags, Node* parm0, Node* parm1, Node* parm2, Node* parm3, Node* parm4, Node* parm5, - Node* parm6, Node* parm7) { + Node* parm6, Node* parm7, + Node* ctrl) { assert(call_addr != NULL, "must not call NULL targets"); // Slow-path call @@ -2513,7 +2657,7 @@ Node* GraphKit::make_runtime_call(int flags, Node* prev_mem = NULL; if (wide_in) { - prev_mem = set_predefined_input_for_runtime_call(call); + prev_mem = set_predefined_input_for_runtime_call(call, ctrl); } else { assert(!wide_out, "narrow in => narrow out"); Node* narrow_mem = memory(adr_type); @@ -3075,8 +3219,17 @@ void GraphKit::guard_init_thread(Node* klass) { init_thread = _gvn.transform(init_thread); Node* cur_thread = _gvn.transform(new ThreadLocalNode()); + Node* cur_wisp = NULL; + if (UseWispMonitor) { + Node* coroutine_offset = _gvn.MakeConX(in_bytes(JavaThread::current_coroutine_offset())); + Node* coroutine_adr = basic_plus_adr(cur_thread, cur_thread, coroutine_offset); + Node* coroutine = make_load(control(), coroutine_adr, TypeRawPtr::BOTTOM, T_ADDRESS, MemNode::unordered); + Node* wisp_thread_offset = _gvn.MakeConX(in_bytes(Coroutine::wisp_thread_offset())); + Node* wisp_thread_adr = basic_plus_adr(coroutine, coroutine, wisp_thread_offset); + cur_wisp = make_load(control(), wisp_thread_adr, TypeRawPtr::BOTTOM, T_ADDRESS, MemNode::unordered); + } - Node* chk = _gvn.transform(new CmpPNode(cur_thread, init_thread)); + Node* chk = _gvn.transform(new CmpPNode(UseWispMonitor ? cur_wisp : cur_thread, init_thread)); Node* tst = _gvn.transform(new BoolNode(chk, BoolTest::eq)); { BuildCutout unless(this, tst, PROB_MAX); @@ -3499,6 +3652,46 @@ Node* GraphKit::insert_mem_bar_volatile(int opcode, int alias_idx, Node* precede return membar; } +#define __ ideal. + +void GraphKit::make_wisp_yield(ciMethod* method) { + assert(EnableCoroutine, "Coroutine is disabled"); + IdealKit ideal(this, true); + Node* no_base = __ top(); + Node* tls = __ thread(); + const int mark_offset = in_bytes(JavaThread::wisp_preempt_offset()); + Node* mark_preempt_addr = __ AddP(no_base, tls, __ ConX(mark_offset)); + Node* mark_preempt_val = __ load(__ ctrl(), mark_preempt_addr, TypeInt::BYTE, T_BYTE, Compile::AliasIdxRaw); + Node* zero = __ ConI(0); + const TypeFunc *call_type = OptoRuntime::yield_method_exit_Type(); + address call_address = CAST_FROM_FN_PTR(address, SharedRuntime::wisp_yield); + const char *call_name = "wisp_yield"; + // Get base of thread-local storage area + Node* thread = _gvn.transform( new ThreadLocalNode() ); + + // Get method + const TypePtr* method_type = TypeMetadataPtr::make(method); + Node *method_node = _gvn.transform( ConNode::make(method_type) ); + + kill_dead_locals(); + + // For some reason, this call reads only raw memory. + const TypePtr* raw_adr_type = TypeRawPtr::BOTTOM; + __ if_then(mark_preempt_val, BoolTest::ne, zero) ; { + // Totally 8 parameters, 2 used, 6 NULL + Node* call = make_runtime_call(RC_NARROW_MEM, + call_type, call_address, + call_name, raw_adr_type, + thread, method_node, + NULL, NULL, NULL, + NULL, NULL, NULL, + __ ctrl()); + __ set_ctrl( _gvn.transform( new ProjNode(call, TypeFunc::Control))); + } __ end_if(); + final_sync(ideal); +} + + //------------------------------shared_lock------------------------------------ // Emit locking code. FastLockNode* GraphKit::shared_lock(Node* obj) { @@ -3600,6 +3793,9 @@ void GraphKit::shared_unlock(Node* box, Node* obj) { unlock->init_req(TypeFunc::Parms + 1, box); unlock = _gvn.transform(unlock)->as_Unlock(); + if (UseWispMonitor && jvms()->has_method()) { + add_safepoint_edges_wisp(unlock); + } Node* mem = reset_memory(); // unlock has no side-effects, sets few values diff --git a/src/hotspot/share/opto/graphKit.hpp b/src/hotspot/share/opto/graphKit.hpp index d815e21956f..422adf1c23e 100644 --- a/src/hotspot/share/opto/graphKit.hpp +++ b/src/hotspot/share/opto/graphKit.hpp @@ -288,6 +288,7 @@ class GraphKit : public Phase { void add_safepoint_edges(SafePointNode* call, bool must_throw = false); + void add_safepoint_edges_wisp(SafePointNode* call); // How many stack inputs does the current BC consume? // And, how does the stack change after the bytecode? // Returns false if unknown. @@ -802,8 +803,10 @@ class GraphKit : public Phase { Node* parm0 = NULL, Node* parm1 = NULL, Node* parm2 = NULL, Node* parm3 = NULL, Node* parm4 = NULL, Node* parm5 = NULL, - Node* parm6 = NULL, Node* parm7 = NULL); + Node* parm6 = NULL, Node* parm7 = NULL, + Node* ctrl = NULL); + void make_wisp_yield(ciMethod* method); Node* sign_extend_byte(Node* in); Node* sign_extend_short(Node* in); diff --git a/src/hotspot/share/opto/machnode.cpp b/src/hotspot/share/opto/machnode.cpp index 09e4cfde429..e6e95b6619f 100644 --- a/src/hotspot/share/opto/machnode.cpp +++ b/src/hotspot/share/opto/machnode.cpp @@ -356,6 +356,14 @@ const class TypePtr *MachNode::adr_type() const { // base of -1 with no particular offset means all of memory if (base == NodeSentinel) return TypePtr::BOTTOM; + // always return TypeRawPtr::BOTTOM for tlsRefetch. + // tlsLoad is special handled in adlc to return bottom type, + // we do not want to hack for tlsRefetch because it is only + // used for complete_monitor_locking_C stub, just check here + if (EnableSteal && base->is_Mach() && base->as_Mach()->ideal_Opcode() == Op_ThreadRefetch) { + return TypeRawPtr::BOTTOM; + } + const Type* t = base->bottom_type(); if (t->isa_narrowoop() && CompressedOops::shift() == 0) { // 32-bit unscaled narrow oop can be the base of any address expression diff --git a/src/hotspot/share/opto/macro.cpp b/src/hotspot/share/opto/macro.cpp index 673784d2aab..754e8a189ab 100644 --- a/src/hotspot/share/opto/macro.cpp +++ b/src/hotspot/share/opto/macro.cpp @@ -2007,7 +2007,13 @@ void PhaseMacroExpand::mark_eliminated_locking_nodes(AbstractLockNode *alock) { } else if (!alock->is_non_esc_obj()) { // Not eliminated or coarsened // Only Lock node has JVMState needed here. // Not that preceding claim is documented anywhere else. - if (alock->jvms() != NULL) { + // But when using wisp, lock node and unlock node both have jvm state. + // The following conditions are used to differentiate lock and unlock. + // 1. alock->jvms() is not NULL. + // 2. alock is_lock() is true. + // 3. option UseWispMonitor is false. + // AbstractLockNode is a lock node if 1 is true and 2 or 3 is ture. + if ((alock->jvms() != NULL) && (alock->is_Lock() || !UseWispMonitor)) { if (alock->as_Lock()->is_nested_lock_region()) { // Mark eliminated related nested locks and unlocks. Node* obj = alock->obj_node(); @@ -2425,9 +2431,13 @@ void PhaseMacroExpand::expand_unlock_node(UnlockNode *unlock) { Node *slow_path = opt_bits_test(ctrl, region, 2, funlock, 0, 0); Node *thread = transform_later(new ThreadLocalNode()); - CallNode *call = make_slow_call((CallNode *) unlock, OptoRuntime::complete_monitor_exit_Type(), - CAST_FROM_FN_PTR(address, SharedRuntime::complete_monitor_unlocking_C), - "complete_monitor_unlocking_C", slow_path, obj, box, thread); + CallNode *call; + if (UseWispMonitor && + ((CallNode *)unlock->jvms() != NULL) && ((CallNode *)unlock->jvms()->has_method())) { + call = make_slow_call( (CallNode *) unlock, OptoRuntime::complete_monitor_exit_Type(), OptoRuntime::complete_wisp_monitor_unlocking_Java(), NULL, slow_path, obj, box, thread); + } else { + call = make_slow_call( (CallNode *) unlock, OptoRuntime::complete_monitor_exit_Type(), CAST_FROM_FN_PTR(address, SharedRuntime::complete_monitor_unlocking_C), "complete_monitor_unlocking_C", slow_path, obj, box, thread); + } call->extract_projections(&_callprojs, false /*separate_io_proj*/, false /*do_asserts*/); assert(_callprojs.fallthrough_ioproj == NULL && _callprojs.catchall_ioproj == NULL && diff --git a/src/hotspot/share/opto/matcher.cpp b/src/hotspot/share/opto/matcher.cpp index 52be4b022ca..45613a354f4 100644 --- a/src/hotspot/share/opto/matcher.cpp +++ b/src/hotspot/share/opto/matcher.cpp @@ -2225,6 +2225,7 @@ bool Matcher::find_shared_visit(MStack& mstack, Node* n, uint opcode, bool& mem_ case Op_FmaF: case Op_FmaVD: case Op_FmaVF: + case Op_ThreadRefetch: // This must be added, otherwise we couldn't match the ThreadRefetchNode. case Op_MacroLogicV: case Op_LoadVectorMasked: case Op_VectorCmpMasked: diff --git a/src/hotspot/share/opto/parse1.cpp b/src/hotspot/share/opto/parse1.cpp index 7819812e1fe..eb9b840d60b 100644 --- a/src/hotspot/share/opto/parse1.cpp +++ b/src/hotspot/share/opto/parse1.cpp @@ -2211,6 +2211,11 @@ void Parse::return_current(Node* value) { if (C->env()->dtrace_method_probes()) { make_dtrace_method_exit(method()); } + // Deadlocks may happen if programs yield at return of synchronized methods. + // Because the yield may come before monitorexit. + if (EnableCoroutine && EnableCoroutineTimeSlice && (!method()->is_synchronized())) { + make_wisp_yield(method()); + } SafePointNode* exit_return = _exits.map(); exit_return->in( TypeFunc::Control )->add_req( control() ); exit_return->in( TypeFunc::I_O )->add_req( i_o () ); diff --git a/src/hotspot/share/opto/runtime.cpp b/src/hotspot/share/opto/runtime.cpp index b5cda23020a..d7d415142d4 100644 --- a/src/hotspot/share/opto/runtime.cpp +++ b/src/hotspot/share/opto/runtime.cpp @@ -102,6 +102,7 @@ address OptoRuntime::_multianewarray5_Java = NULL; address OptoRuntime::_multianewarrayN_Java = NULL; address OptoRuntime::_vtable_must_compile_Java = NULL; address OptoRuntime::_complete_monitor_locking_Java = NULL; +address OptoRuntime::_complete_wisp_monitor_unlocking_Java = NULL; address OptoRuntime::_monitor_notify_Java = NULL; address OptoRuntime::_monitor_notifyAll_Java = NULL; address OptoRuntime::_rethrow_Java = NULL; @@ -145,6 +146,7 @@ bool OptoRuntime::generate(ciEnv* env) { gen(env, _multianewarray5_Java , multianewarray5_Type , multianewarray5_C , 0 , true, false); gen(env, _multianewarrayN_Java , multianewarrayN_Type , multianewarrayN_C , 0 , true, false); gen(env, _complete_monitor_locking_Java , complete_monitor_enter_Type , SharedRuntime::complete_monitor_locking_C, 0, false, false); + gen(env, _complete_wisp_monitor_unlocking_Java , complete_monitor_enter_Type , SharedRuntime::complete_wisp_monitor_unlocking_C, 0, false, false); gen(env, _monitor_notify_Java , monitor_notify_Type , monitor_notify_C , 0 , false, false); gen(env, _monitor_notifyAll_Java , monitor_notify_Type , monitor_notifyAll_C , 0 , false, false); gen(env, _rethrow_Java , rethrow_Type , rethrow_C , 2 , true , true ); @@ -1574,6 +1576,21 @@ JRT_ENTRY_NO_ASYNC(void, OptoRuntime::register_finalizer(oopDesc* obj, JavaThrea InstanceKlass::register_finalizer(instanceOop(obj), CHECK); JRT_END + +const TypeFunc *OptoRuntime::yield_method_exit_Type() { + // create input type (domain) + const Type **fields = TypeTuple::fields(2); + fields[TypeFunc::Parms+0] = TypeRawPtr::BOTTOM; // Thread-local storage + fields[TypeFunc::Parms+1] = TypeMetadataPtr::BOTTOM; // Method*; Method we are entering + const TypeTuple *domain = TypeTuple::make(TypeFunc::Parms+2,fields); + + // create result type (range) + fields = TypeTuple::fields(0); + + const TypeTuple *range = TypeTuple::make(TypeFunc::Parms+0,fields); + + return TypeFunc::make(domain,range); +} //----------------------------------------------------------------------------- NamedCounter * volatile OptoRuntime::_named_counters = NULL; diff --git a/src/hotspot/share/opto/runtime.hpp b/src/hotspot/share/opto/runtime.hpp index 26f80acac76..f1b90708dde 100644 --- a/src/hotspot/share/opto/runtime.hpp +++ b/src/hotspot/share/opto/runtime.hpp @@ -143,6 +143,7 @@ class OptoRuntime : public AllStatic { static address _multianewarrayN_Java; static address _vtable_must_compile_Java; static address _complete_monitor_locking_Java; + static address _complete_wisp_monitor_unlocking_Java; static address _rethrow_Java; static address _monitor_notify_Java; static address _monitor_notifyAll_Java; @@ -173,6 +174,7 @@ class OptoRuntime : public AllStatic { // Slow-path Locking and Unlocking static void complete_monitor_locking_C(oopDesc* obj, BasicLock* lock, JavaThread* thread); static void complete_monitor_unlocking_C(oopDesc* obj, BasicLock* lock, JavaThread* thread); + static void complete_wisp_monitor_unlocking_C(oopDesc* obj, BasicLock* lock, JavaThread* thread); static void monitor_notify_C(oopDesc* obj, JavaThread* current); static void monitor_notifyAll_C(oopDesc* obj, JavaThread* current); @@ -221,6 +223,7 @@ class OptoRuntime : public AllStatic { static address multianewarrayN_Java() { return _multianewarrayN_Java; } static address vtable_must_compile_stub() { return _vtable_must_compile_Java; } static address complete_monitor_locking_Java() { return _complete_monitor_locking_Java; } + static address complete_wisp_monitor_unlocking_Java() { return _complete_wisp_monitor_unlocking_Java; } static address monitor_notify_Java() { return _monitor_notify_Java; } static address monitor_notifyAll_Java() { return _monitor_notifyAll_Java; } @@ -311,6 +314,9 @@ class OptoRuntime : public AllStatic { static const TypeFunc* dtrace_method_entry_exit_Type(); static const TypeFunc* dtrace_object_alloc_Type(); + // Yield support + static const TypeFunc* yield_method_exit_Type(); + private: static NamedCounter * volatile _named_counters; diff --git a/src/hotspot/share/prims/jni.cpp b/src/hotspot/share/prims/jni.cpp index a90835852ad..a8e2abf4dff 100644 --- a/src/hotspot/share/prims/jni.cpp +++ b/src/hotspot/share/prims/jni.cpp @@ -3661,30 +3661,6 @@ static jint JNI_CreateJavaVM_inner(JavaVM **vm, void **penv, void *args) { VMError::controlled_crash(ErrorHandlerTest); } #endif - if (EnableCoroutine) { - JavaThread* __the_thread__ = thread; - HandleMark hm(THREAD); - Handle obj(THREAD, thread->threadObj()); - JavaValue result(T_VOID); - - if (vmClasses::java_dyn_CoroutineSupport_klass() != NULL) { - InstanceKlass::cast(vmClasses::Class_klass())->initialize(CHECK_0); - InstanceKlass::cast(vmClasses::java_dyn_CoroutineSupport_klass())->initialize(CHECK_0); - JavaCalls::call_virtual(&result, - obj, - vmClasses::Thread_klass(), - vmSymbols::initializeCoroutineSupport_method_name(), - vmSymbols::void_method_signature(), - THREAD); - if (THREAD->has_pending_exception()) { - Handle exception(THREAD, THREAD->pending_exception()); - java_lang_Throwable::print_stack_trace(exception, tty); - THREAD->clear_pending_exception(); - vm_abort(false); - } - } - } - // Since this is not a JVM_ENTRY we have to set the thread state manually before leaving. ThreadStateTransition::transition(thread, _thread_in_vm, _thread_in_native); @@ -3909,6 +3885,10 @@ static jint attach_current_thread(JavaVM *vm, void **penv, void *_args, bool dae *(JNIEnv**)penv = thread->jni_environment(); + if (EnableCoroutine) { + Coroutine::initialize_coroutine_support(JavaThread::current()); + } + // Now leaving the VM, so change thread_state. This is normally automatically taken care // of in the JVM_ENTRY. But in this situation we have to do it manually. Notice, that by // using ThreadStateTransition::transition, we do a callback to the safepoint code if diff --git a/src/hotspot/share/prims/jvm.cpp b/src/hotspot/share/prims/jvm.cpp index 3863885ae48..1f48f7fa299 100644 --- a/src/hotspot/share/prims/jvm.cpp +++ b/src/hotspot/share/prims/jvm.cpp @@ -606,6 +606,13 @@ JVM_END JVM_ENTRY(void, JVM_MonitorWait(JNIEnv* env, jobject handle, jlong ms)) Handle obj(THREAD, JNIHandles::resolve_non_null(handle)); JavaThreadInObjectWaitState jtiows(thread, ms != 0); + + Thread* t = THREAD; + WispPostStealHandleUpdateMark w(thread, t, env, __tiv, __hm, &jtiows); + + // Coroutine work steal support + EnableStealMark p(THREAD); + if (JvmtiExport::should_post_monitor_wait()) { JvmtiExport::post_monitor_wait(thread, obj(), ms); @@ -2844,23 +2851,6 @@ static void thread_entry(JavaThread* thread, TRAPS) { Handle obj(THREAD, thread->threadObj()); JavaValue result(T_VOID); - if (EnableCoroutine && vmClasses::java_dyn_CoroutineSupport_klass() != NULL) { - InstanceKlass::cast(vmClasses::Class_klass())->initialize(CHECK); - InstanceKlass::cast(vmClasses::java_dyn_CoroutineSupport_klass())->initialize(CHECK); - JavaCalls::call_virtual(&result, - obj, - vmClasses::Thread_klass(), - vmSymbols::initializeCoroutineSupport_method_name(), - vmSymbols::void_method_signature(), - THREAD); - if (THREAD->has_pending_exception()) { - Handle exception(THREAD, THREAD->pending_exception()); - java_lang_Throwable::print_stack_trace(exception, tty); - THREAD->clear_pending_exception(); - vm_abort(false); - } - } - JavaCalls::call_virtual(&result, obj, vmClasses::Thread_klass(), @@ -3113,6 +3103,27 @@ JVM_ENTRY(void, JVM_Interrupt(JNIEnv* env, jobject jthread)) } JVM_END +JVM_ENTRY(jboolean, JVM_IsInNative(JNIEnv* env, jobject jthread)) + assert(EnableCoroutine, "Coroutine is disabled"); + oop java_thread = JNIHandles::resolve_non_null(jthread); + MutexLocker ml(thread->threadObj() == java_thread ? NULL : Threads_lock); + // We need to re-resolve the java_thread, since a GC might have happened during the + // acquire of the lock + JavaThread* thr = java_lang_Thread::thread(JNIHandles::resolve_non_null(jthread)); + return thr != NULL && thr->thread_state() == _thread_in_native; +JVM_END + +JVM_ENTRY(jboolean, JVM_CheckAndClearNativeInterruptForWisp(JNIEnv* env, jobject task, jobject jthread)) + // here, maybe the thread is in `Thread.start()`, the eetop is not settled, so we should also block the + // condition with `th` is null. + assert(EnableCoroutine, "Coroutine is disabled"); + JavaThread *th = java_lang_Thread::thread(JNIHandles::resolve_non_null(jthread)); + if (th != NULL) { + return (jboolean)clear_interrupt_for_wisp(th); + } else { + return (jboolean)false; + } +JVM_END // Return true iff the current thread has locked the object passed in @@ -3502,6 +3513,10 @@ jclass find_class_from_class_loader(JNIEnv* env, Symbol* name, jboolean init, // Method /////////////////////////////////////////////////////////////////////////////////////////// JVM_ENTRY(jobject, JVM_InvokeMethod(JNIEnv *env, jobject method, jobject obj, jobjectArray args0)) + // Coroutine work steal support + Thread* t = THREAD; + WispPostStealHandleUpdateMark w(thread, t, env, __tiv, __hm); + Handle method_handle; if (thread->stack_overflow_state()->stack_available((address) &method_handle) >= JVMInvokeMethodSlack) { method_handle = Handle(THREAD, JNIHandles::resolve(method)); @@ -3526,6 +3541,10 @@ JVM_END JVM_ENTRY(jobject, JVM_NewInstanceFromConstructor(JNIEnv *env, jobject c, jobjectArray args0)) + // Coroutine work steal support + Thread* t = THREAD; + WispPostStealHandleUpdateMark w(thread, t, env, __tiv, __hm); + oop constructor_mirror = JNIHandles::resolve(c); objArrayHandle args(THREAD, objArrayOop(JNIHandles::resolve(args0))); oop result = Reflection::invoke_constructor(constructor_mirror, args, CHECK_NULL); @@ -3843,6 +3862,30 @@ JVM_ENTRY(jobjectArray, JVM_GetEnclosingMethodInfo(JNIEnv *env, jclass ofClass)) } JVM_END +JVM_ENTRY(void, JVM_SetWispTask(JNIEnv* env, jclass klass, jlong coroutinePtr, jint task_id, jobject task, jobject engine)) + assert(EnableCoroutine, "Coroutine is disabled"); + Coroutine* coro = (Coroutine*)coroutinePtr; + coro->set_wisp_task_id(task_id); + coro->set_wisp_engine(JNIHandles::resolve_non_null(engine)); + coro->set_wisp_task(JNIHandles::resolve_non_null(task)); +JVM_END + +JVM_ENTRY(jint, JVM_GetProxyUnpark(JNIEnv* env, jclass klass, jintArray res)) + assert(EnableCoroutine, "Coroutine is disabled"); + return WispThread::get_proxy_unpark(res); +JVM_END + +JVM_ENTRY(void, JVM_MarkPreempt(JNIEnv* env, jclass klass, jobject threadObj)) + assert(EnableCoroutine, "Coroutine is disabled"); + //Use lock to prevent deleting thr when we do the update on it. + MutexLocker mu(Threads_lock); + JavaThread* thr = java_lang_Thread::thread(JNIHandles::resolve_non_null(threadObj)); + + if (thr != NULL && !thr->is_terminated()) { + thr->set_wisp_preempt(true); + } +JVM_END + // Returns an array of java.lang.String objects containing the input arguments to the VM. JVM_ENTRY(jobjectArray, JVM_GetVmArguments(JNIEnv *env)) ResourceMark rm(THREAD); diff --git a/src/hotspot/share/prims/jvmtiGetLoadedClasses.cpp b/src/hotspot/share/prims/jvmtiGetLoadedClasses.cpp index a12b1f97181..bb4bad03672 100644 --- a/src/hotspot/share/prims/jvmtiGetLoadedClasses.cpp +++ b/src/hotspot/share/prims/jvmtiGetLoadedClasses.cpp @@ -124,7 +124,7 @@ JvmtiGetLoadedClasses::getClassLoaderClasses(JvmtiEnv *env, jobject initiatingLo // To get a consistent list of classes we need MultiArray_lock to ensure // array classes aren't created during this walk. MutexLocker ma(MultiArray_lock); - MutexLocker sd(SystemDictionary_lock); + SystemDictLocker sd(JavaThread::current(), SystemDictionary_lock); oop loader = JNIHandles::resolve(initiatingLoader); // All classes loaded from this loader as initiating loader are // requested, so only need to walk this loader's ClassLoaderData diff --git a/src/hotspot/share/prims/jvmtiThreadState.cpp b/src/hotspot/share/prims/jvmtiThreadState.cpp index e61241fdd63..4fc3a0f4c2a 100644 --- a/src/hotspot/share/prims/jvmtiThreadState.cpp +++ b/src/hotspot/share/prims/jvmtiThreadState.cpp @@ -282,6 +282,9 @@ int JvmtiThreadState::cur_stack_depth() { if (!is_interp_only_mode() || _cur_stack_depth == UNKNOWN_STACK_DEPTH) { _cur_stack_depth = count_frames(); } else { + if (EnableCoroutine) { + _cur_stack_depth = count_frames(); // update debug stack depth, for switchToAndExit + } #ifdef ASSERT if (EnableJVMTIStackDepthAsserts) { // heavy weight assert diff --git a/src/hotspot/share/prims/unsafe.cpp b/src/hotspot/share/prims/unsafe.cpp index 0f3bbae9dc8..9e2b1485474 100644 --- a/src/hotspot/share/prims/unsafe.cpp +++ b/src/hotspot/share/prims/unsafe.cpp @@ -943,9 +943,8 @@ JVM_ENTRY(jlong, CoroutineSupport_createCoroutine(JNIEnv* env, jclass klass, job return (jlong)coro; JVM_END - -JVM_ENTRY(jboolean, CoroutineSupport_isDisposable(JNIEnv* env, jclass klass, jlong coroutineLong)) - DEBUG_CORO_PRINT("CoroutineSupport_isDisposable\n"); +JVM_ENTRY(jboolean, CoroutineSupport_testDisposableAndTryReleaseStack(JNIEnv* env, jclass klass, jlong coroutineLong)) + DEBUG_CORO_PRINT("CoroutineSupport_testDisposableAndTryReleaseStack\n"); Coroutine* coro = (Coroutine*)coroutineLong; assert(coro != NULL, "cannot free NULL coroutine"); assert(!coro->is_thread_coroutine(), "cannot free thread coroutine"); @@ -968,6 +967,41 @@ JVM_ENTRY(jobject, CoroutineSupport_cleanupCoroutine(JNIEnv* env, jclass klass)) return NULL; JVM_END +JVM_ENTRY(void, CoroutineSupport_setWispBooted(JNIEnv* env, jclass klass)) + DEBUG_CORO_PRINT("CoroutineSupport_setWispBooted\n"); + WispThread::set_wisp_booted(thread); +JVM_END + +JVM_ENTRY(jboolean, CoroutineSupport_stealCoroutine(JNIEnv* env, jclass klass, jlong coroPtr)) + // We've already locked the target's thread + // and source's thread. target_thread->coroutine_list()s have + // no way to be changed during this process. + // + // The lock will also block coroutine switch operation, + // so we must finish the steal operation as soon as possible. + Coroutine* coro = (Coroutine*) coroPtr; + if (!EnableSteal || coro == NULL || coro->enable_steal_count() != coro->java_call_counter()) { + return false; // an Exception throws and the coroutine being stealed is exited + } + assert(coro->thread() != thread, "steal from self"); + assert(coro->state() != Coroutine::_current, "running"); + coro->remove_from_list(coro->thread()->coroutine_list()); + coro->insert_into_list(thread->coroutine_list()); + // change thread logic + if (coro->last_handle_mark() != NULL) { + coro->last_handle_mark()->change_thread_for_wisp(thread); + } + coro->change_thread_for_wisp(thread); + coro->set_thread(thread); + if (UseWispMonitor) { + if (coro->wisp_thread()) { + coro->wisp_thread()->change_thread(thread); + coro->set_wisp_engine(thread->current_coroutine()->wisp_engine()); + } + } + return true; +JVM_END + /// JVM_RegisterUnsafeMethods #define ADR "J" @@ -1031,8 +1065,8 @@ static JNINativeMethod jdk_internal_misc_Unsafe_methods[] = { {CC "compareAndExchangeInt", CC "(" OBJ "J""I""I"")I", FN_PTR(Unsafe_CompareAndExchangeInt)}, {CC "compareAndExchangeLong", CC "(" OBJ "J""J""J"")J", FN_PTR(Unsafe_CompareAndExchangeLong)}, - {CC "park", CC "(ZJ)V", FN_PTR(Unsafe_Park)}, - {CC "unpark", CC "(" OBJ ")V", FN_PTR(Unsafe_Unpark)}, + {CC "park0", CC "(ZJ)V", FN_PTR(Unsafe_Park)}, + {CC "unpark0", CC "(" OBJ ")V", FN_PTR(Unsafe_Unpark)}, {CC "getLoadAverage0", CC "([DI)I", FN_PTR(Unsafe_GetLoadAverage0)}, @@ -1053,16 +1087,19 @@ static JNINativeMethod jdk_internal_misc_Unsafe_methods[] = { #define COBA "Ljava/dyn/CoroutineBase;" JNINativeMethod coroutine_support_methods[] = { - {CC"getThreadCoroutine", CC "()J", FN_PTR(CoroutineSupport_getThreadCoroutine)}, - {CC"createCoroutine", CC "(" COBA "J)J", FN_PTR(CoroutineSupport_createCoroutine)}, - {CC"isDisposable", CC "(J)Z", FN_PTR(CoroutineSupport_isDisposable)}, {CC"switchTo", CC "(" COBA COBA ")V", FN_PTR(CoroutineSupport_switchTo)}, {CC"switchToAndTerminate", CC "(" COBA COBA ")V", FN_PTR(CoroutineSupport_switchToAndTerminate)}, {CC"switchToAndExit", CC "(" COBA COBA ")V", FN_PTR(CoroutineSupport_switchToAndExit)}, + {CC"getThreadCoroutine", CC "()J", FN_PTR(CoroutineSupport_getThreadCoroutine)}, + {CC"createCoroutine", CC "(" COBA "J)J", FN_PTR(CoroutineSupport_createCoroutine)}, + {CC"testDisposableAndTryReleaseStack", + CC "(J)Z", FN_PTR(CoroutineSupport_testDisposableAndTryReleaseStack)}, {CC"cleanupCoroutine", CC "()" COBA, FN_PTR(CoroutineSupport_cleanupCoroutine)}, + {CC"setWispBooted", CC "()V", FN_PTR(CoroutineSupport_setWispBooted)}, + {CC"stealCoroutine", CC "(J)Z", FN_PTR(CoroutineSupport_stealCoroutine)}, }; -#define COMPILE_CORO_METHODS_FROM (3) +#define COMPILE_CORO_METHODS_BEFORE (3) #undef COBA @@ -1105,10 +1142,11 @@ JVM_ENTRY(void, JVM_RegisterCoroutineSupportMethods(JNIEnv *env, jclass corocls) env->RegisterNatives(corocls, coroutine_support_methods + i, 1); if (env->ExceptionOccurred()) { tty->print_cr("Warning: Coroutine classes not found (%i)", i); + env->ExceptionDescribe(); vm_exit(1); } } - for (int i=COMPILE_CORO_METHODS_FROM; iGetStaticMethodID(corocls, coroutine_support_methods[i].name, coroutine_support_methods[i].signature); { ThreadInVMfromNative tivfn(thread); diff --git a/src/hotspot/share/runtime/arguments.cpp b/src/hotspot/share/runtime/arguments.cpp index 2341036b44f..7481f536d5e 100644 --- a/src/hotspot/share/runtime/arguments.cpp +++ b/src/hotspot/share/runtime/arguments.cpp @@ -443,6 +443,23 @@ void Arguments::init_wisp_system_properties() { PropertyList_add(&_system_properties, new SystemProperty("com.alibaba.coroutine.enableCoroutine", EnableCoroutine ? "true" : "false", false)); + if (UseWisp2) { + PropertyList_add(&_system_properties, + new SystemProperty("com.alibaba.wisp.version", + "2", false)); + // Props set by user -D flags would override those default values. + PropertyList_add(&_system_properties, + new SystemProperty("com.alibaba.wisp.transparentWispSwitch", + "true", true)); + PropertyList_add(&_system_properties, + new SystemProperty("com.alibaba.wisp.enableThreadAsWisp", + "true", true)); + if (Arguments::get_property("com.alibaba.wisp.allThreadAsWisp") == NULL) { + PropertyList_add(&_system_properties, + new SystemProperty("com.alibaba.wisp.allThreadAsWisp", + "true", true)); + } + } } /* @@ -3942,6 +3959,38 @@ jint Arguments::parse(const JavaVMInitArgs* initial_cmd_args) { return JNI_EINVAL; } + if (UseWisp2) { + // check Compatibility + if (!EnableCoroutine && FLAG_IS_CMDLINE(EnableCoroutine)) { + warning("Wisp2 needs to enable -XX:+EnableCoroutine" + "; ignoring -XX:-EnableCoroutine." ); + } + if (!UseWispMonitor && FLAG_IS_CMDLINE(UseWispMonitor)) { + warning("Wisp2 needs to enable -XX:+UseWispMonitor" + "; ignoring -XX:-UseWispMonitor." ); + } + if (UseBiasedLocking && FLAG_IS_CMDLINE(UseBiasedLocking)) { + warning("Biased Locking is not supported with Wisp2" + "; ignoring UseBiasedLocking flag." ); + } + // Turn on -XX:+EnableCoroutine, -XX:+UseWispMonitor + EnableCoroutine = true; + UseWispMonitor = true; + // Turn off -XX:-UseBiasedLocking + UseBiasedLocking = false; + } else { + if (EnableCoroutine) { + if (UseBiasedLocking && FLAG_IS_CMDLINE(UseBiasedLocking)) { + warning("Biased Locking is not supported with Wisp" + "; ignoring UseBiasedLocking flag." ); + } + UseBiasedLocking = false; + } + } + if (!EnableCoroutine) { + EnableSteal = false; + } + // Set object alignment values. set_object_alignment(); diff --git a/src/hotspot/share/runtime/coroutine.cpp b/src/hotspot/share/runtime/coroutine.cpp index 1f2cfe6ce76..de6514612b7 100644 --- a/src/hotspot/share/runtime/coroutine.cpp +++ b/src/hotspot/share/runtime/coroutine.cpp @@ -23,14 +23,22 @@ */ #include "precompiled.hpp" +//#include "prims/privilegedStack.hpp" #include "classfile/vmSymbols.hpp" +#include "interpreter/linkResolver.hpp" #include "runtime/coroutine.hpp" #include "runtime/globals.hpp" #include "runtime/handles.inline.hpp" +#include "runtime/interfaceSupport.inline.hpp" #include "runtime/javaCalls.hpp" +#include "runtime/jniHandles.inline.hpp" +#include "runtime/objectMonitor.hpp" +#include "runtime/objectMonitor.inline.hpp" #include "runtime/os.inline.hpp" #include "runtime/stackFrameStream.inline.hpp" #include "runtime/thread.inline.hpp" +#include "runtime/vframe.hpp" +#include "services/threadService.hpp" #ifdef TARGET_ARCH_x86 # include "vmreg_x86.inline.hpp" #endif @@ -40,7 +48,7 @@ #ifdef TARGET_ARCH_zero # include "vmreg_zero.inline.hpp" #endif -#include "jniHandles.inline.hpp" + #ifdef _WINDOWS @@ -115,18 +123,34 @@ Coroutine* Coroutine::create_thread_coroutine(JavaThread* thread, CoroutineStack coro->_last_handle_mark = NULL; coro->_active_handles = thread->active_handles(); coro->_metadata_handles = NULL; + coro->_thread_status = JavaThreadStatus::RUNNABLE; coro->_java_call_counter = 0; #if defined(_WINDOWS) coro->_last_SEH = NULL; #endif + // coro->_privileged_stack_top = NULL; + coro->_wisp_thread = UseWispMonitor ? new WispThread(coro) : NULL; + coro->_wisp_engine = NULL; + coro->_wisp_task = NULL; + coro->_wisp_task_id = WISP_ID_NOT_SET; + coro->_enable_steal_count = 1; + coro->_wisp_post_steal_resource_area = NULL; + thread->set_current_coroutine(coro); return coro; } +/** + * The initial value for corountine' active handles, which will be replaced with a real one + * when the coroutine invokes call_virtual (before running any real logic). + */ +static JNIHandleBlock* shared_empty_JNIHandleBlock = JNIHandleBlock::allocate_block(); + Coroutine* Coroutine::create_coroutine(JavaThread* thread, CoroutineStack* stack, oop coroutineObj) { Coroutine* coro = new Coroutine(); if (coro == NULL) { return NULL; } + intptr_t** d = (intptr_t**)stack->stack_base(); *(--d) = NULL; // Make it to be 16 bytes(original is 8*5=40 bytes) aligned which be required by some instruction likes movaps otherwise we will incur crash. *(--d) = NULL; @@ -146,17 +170,33 @@ Coroutine* Coroutine::create_coroutine(JavaThread* thread, CoroutineStack* stack coro->_resource_area = NULL; coro->_handle_area = NULL; coro->_last_handle_mark = NULL; - coro->_active_handles = thread->active_handles(); + coro->_active_handles = shared_empty_JNIHandleBlock; coro->_metadata_handles = NULL; + coro->_thread_status = JavaThreadStatus::RUNNABLE; coro->_java_call_counter = 0; #if defined(_WINDOWS) coro->_last_SEH = NULL; #endif + // coro->_privileged_stack_top = NULL; + coro->_wisp_thread = UseWispMonitor ? new WispThread(coro) : NULL; + coro->_wisp_engine = NULL; + coro->_wisp_task = NULL; + coro->_wisp_task_id = WISP_ID_NOT_SET; + // if coro->_enable_steal_count == coro->_java_call_counter is true, we can do work steal. + // when a coroutine starts, the `_java_call_counter` is 0, + // then it will call java method `Coroutine.startInternal()` and then `_java_call_counter` is 1. + // so we set `_enable_steal_count` to 1 which means this coroutine can be stolen when it starts. + coro->_enable_steal_count = 1; + coro->_wisp_post_steal_resource_area = new (mtWisp) WispResourceArea(coro, 32); return coro; } Coroutine::~Coroutine() { remove_from_list(_thread->coroutine_list()); + if (_wisp_thread != NULL) { + delete _wisp_thread; + } + delete _wisp_post_steal_resource_area; if (!_is_thread_coroutine && _state != Coroutine::_created) { assert(_resource_area != NULL, "_resource_area is NULL"); assert(_handle_area != NULL, "_handle_area is NULL"); @@ -201,10 +241,18 @@ class oops_do_Closure: public FrameClosure { void Coroutine::oops_do(OopClosure* f, CodeBlobClosure* cf) { oops_do_Closure fc(f, cf); frames_do(&fc); - if (_state == _onstack &&_handle_area != NULL) { + if (_state == _onstack) { + assert(_handle_area != NULL, "_onstack coroutine should have _handle_area"); DEBUG_CORO_ONLY(tty->print_cr("collecting handle area %08x", _handle_area)); _handle_area->oops_do(f); _active_handles->oops_do(f); + // if (_privileged_stack_top != NULL) { + // _privileged_stack_top->oops_do(f); + // } + } + if (_wisp_task != NULL) { + f->do_oop((oop*) &_wisp_engine); + f->do_oop((oop*) &_wisp_task); } } @@ -258,6 +306,10 @@ bool Coroutine::is_disposable() { return _handle_area == NULL; } +void Coroutine::set_wisp_engine(oop x) { + _wisp_engine = x; +} + CoroutineStack* CoroutineStack::create_thread_stack(JavaThread* thread) { CoroutineStack* stack = new CoroutineStack(0); @@ -280,7 +332,7 @@ CoroutineStack* CoroutineStack::create_stack(JavaThread* thread, intptr_t size/* default_size = true; } - uint reserved_pages = StackShadowPages + StackRedPages + StackYellowPages; + uint reserved_pages = StackShadowPages + StackRedPages + StackYellowPages + + StackReservedPages; uintx real_stack_size = size + (reserved_pages * os::vm_page_size()); uintx reserved_size = align_up(real_stack_size, os::vm_allocation_granularity()); @@ -302,7 +354,7 @@ CoroutineStack* CoroutineStack::create_stack(JavaThread* thread, intptr_t size/* if (os::uses_stack_guard_pages()) { address low_addr = stack->stack_base() - stack->stack_size(); - size_t len = (StackYellowPages + StackRedPages) * os::vm_page_size(); + size_t len = (StackYellowPages + StackRedPages + StackReservedPages) * os::vm_page_size(); bool allocate = os::must_commit_stack_guard_pages(); @@ -360,5 +412,558 @@ frame CoroutineStack::last_frame(Coroutine* coro, RegisterMap& map) const { address pc = ((address*)_last_sp)[1]; intptr_t* sp = ((intptr_t*)_last_sp) + 2; + map.set_location(rbp->as_VMReg(), (address)_last_sp); + map.set_include_argument_oops(false); + return frame(sp, fp, pc); } + +void Coroutine::print_stack_on(outputStream* st) { + if (_state == Coroutine::_onstack) { + oop thread_obj = NULL; + st->print("\n - Coroutine [%p]", this); + if (_wisp_task != NULL) { + thread_obj = com_alibaba_wisp_engine_WispTask::get_threadWrapper(_wisp_task); + char buf[128] = ""; + if (thread_obj != NULL) { + oop name = java_lang_Thread::name(thread_obj); + if (name != NULL) { + java_lang_String::as_utf8_string(name, buf, sizeof(buf)); + } + } + st->print(" \"%s\" #%d active=%d steal=%d steal_fail=%d", buf, + com_alibaba_wisp_engine_WispTask::get_id(_wisp_task), + com_alibaba_wisp_engine_WispTask::get_activeCount(_wisp_task), + com_alibaba_wisp_engine_WispTask::get_stealCount(_wisp_task), + com_alibaba_wisp_engine_WispTask::get_stealFailureCount(_wisp_task)); + } // else, we're only using the JKU part + st->print("\n"); + + ResourceMark rm; + RegisterMap reg_map(_thread); + frame last_frame = _stack->last_frame(this, reg_map); + + int count = 0; + JavaThread* t = UseWispMonitor ? _wisp_thread : _thread; + for (vframe* vf = vframe::new_vframe(&last_frame, ®_map, t); vf; vf = vf->sender()) { + if (vf->is_java_frame()) { + javaVFrame* jvf = javaVFrame::cast(vf); + java_lang_Throwable::print_stack_element(st, jvf->method(), jvf->bci()); + + if (UseWispMonitor && JavaMonitorsInStackTrace) { + // t is WispThread + t->set_threadObj(thread_obj); + // ensure thread()->current_park_blocker() fetch the correct thread_obj + jvf->print_lock_info_on(st, count++); + } + } + } + } +} + + +// ---------- lock support ----------- +bool WispThread::_wisp_booted = false; +Method* WispThread::parkMethod = NULL; +Method* WispThread::unparkMethod = NULL; +GrowableArray* WispThread::_proxy_unpark = NULL; + +void WispThread::set_wisp_booted(JavaThread* thread) { + // In JDK11, it introduces metaspace compact. Storing the method isn't safe. + // The flow should be changed. + CallInfo callinfo; + LinkInfo link_info(vmClasses::com_alibaba_wisp_engine_WispTask_klass(), vmSymbols::park_name(), vmSymbols::long_void_signature()); + LinkResolver::resolve_static_call(callinfo, link_info, true, thread); + parkMethod = callinfo.selected_method(); + assert(parkMethod.not_null(), "should have thrown exception"); + + LinkInfo link_info_unpark(vmClasses::com_alibaba_wisp_engine_WispTask_klass(), vmSymbols::unparkById_name(), vmSymbols::int_void_signature()); + LinkResolver::resolve_static_call(callinfo, link_info_unpark, true, thread); + unparkMethod = callinfo.selected_method(); + assert(unparkMethod.not_null(), "should have thrown exception"); + + if (UseWispMonitor) { + _proxy_unpark = new (ResourceObj::C_HEAP, mtWisp) GrowableArray(30); + if (!AlwaysLockClassLoader) { + SystemDictionary::system_dict_lock_change(thread); + } + } + + _wisp_booted = true; // set after parkMethod != null +} + +/* + * Avoid coroutine switch in the following scenarios: + * + * - _wisp_booted: + * We guarantee the classes referenced by WispTask.park(called in WispThread::park in native) + * are already loaded after _wisp_booted is set(as true). Otherwise it might result in loading class during execution of WispTask.park. + * Coroutine switch caused by object monitors in class loading might lead to recursive deadlock. + * + * - !com_alibaba_wisp_engine_WispEngine::is_critical(_coroutine->wisp_engine()): + * If the program is already running in kernel code of wisp engine(marked by WispEngine.isInCritical at Java level), we don't expect + * the switch while coroutine running into 'synchronized' block which is heavily used by Java NIO library. + * Otherwise, it might lead to potential recursive deadlock. + * + * - monitor->object() != java_lang_ref_Reference::pending_list_lock(): + * pending_list_lock(PLL) is special ObjectMonitor used in GC vm operation. + * if we treated it as normal monitor(T10965418) + * - 'dead lock' in jni_critical case: + * Given coroutine A, B running in the same thread, + * 1. coroutine A called jni_ReleasePrimitiveArrayCritical, and triggered GC + * 1.1. VM_GC_Operation::doit_prologue() -> acquired PLL and yielded to coroutine B(because of contending) + * 2. If coroutine B called jni_GetPrimitiveArrayCritical at this moment, it would be blocked by GC flag(jni_lock) + * 3. Unfortunately, A doesn't have any chance to release PLL(acquired in 1.1). As a result, the process is suspended. + * - if we disable switch in InstanceRefKlass::acquire_pending_list_lock(ObjectSynchronizer::fast_enter) + * Given coroutine A, B running in the same thread, + * 1. coroutine A attempted to acquire PLL and yielded out (because of contending) + * 2. coroutine B in VM_GC_Operation::doit_prologue() attempted to acquire PLL(which is triggered by allocation failure), gets blocked + * as we disable the switch in InstanceRefKlass::acquire_pending_list_lock. + * 3. Another thread unparked A, but unfortunately the engine is already blocked. + * So , totally disable the switch in PLL is better solution(monitor->object() != java_lang_ref_Reference::pending_list_lock()). + * +*/ +void WispThread::before_enqueue(ObjectMonitor* monitor, ObjectWaiter* ow) { + JavaThreadState st = _thread->_thread_state; + if (st != _thread_in_vm) { + assert(st == _thread_blocked, "_thread_state should be _thread_blocked"); + ThreadStateTransition::transition(_thread, st, _thread_in_vm); + } + bool is_sd_lock = monitor->object() == SystemDictionary_lock->obj(); + if (_wisp_booted && _coroutine->wisp_task_id() != Coroutine::WISP_ID_NOT_SET + // For java threads, including JvmtiAgentThread, ServiceThread, + // SurrogateLockerThread, CompilerThread which are only used in JVM, + // because we don't initialize co-routine stuff for them, the initial + // value of _coroutine->wisp_task_id() for them should be Coroutine::WISP_ID_NOT_SET. + && !com_alibaba_wisp_engine_WispEngine::in_critical(_coroutine->wisp_engine()) + //&& monitor->object() != java_lang_ref_Reference::pending_list_lock() + && !_thread->in_critical() + && Compile_lock->owner() != _thread + // case holding Compile_lock, we will park for lock contention + // not for monitor->wait() + // so we'll be unparked immediately. It's safe to block the JavaThread + && !(is_sd_lock && ow->TState != ObjectWaiter::TS_WAIT) + // case parking for fetching SystemDictionary_lock fails, DO NOT schedule + // otherwise we'll encounter DEADLOCK because of fetching CompilerQeueu_lock in java + ) { + ow->_using_wisp_park = true; + ow->_park_wisp_id = _coroutine->wisp_task_id(); + com_alibaba_wisp_engine_WispTask::set_jvmParkStatus(_coroutine->wisp_task(), 0); //reset + } else { + ow->_using_wisp_park = false; + ow->_park_wisp_id = Coroutine::WISP_ID_NOT_SET; + } + // when we're operating on SystemDictionary_lock, we're running jvm codes, + // if the unpark was lazy dispatched, the thread may block on a jvm Monitor + // before the unpark disptach. + // So we proxy all unpark operations on the SystemDictionary_lock. + ow->_proxy_wisp_unpark = is_sd_lock; + ow->_timeout = is_sd_lock && ow->TState != ObjectWaiter::TS_WAIT ? 1 : -1; + if (st != _thread_in_vm) { + ThreadStateTransition::transition(_thread, _thread_in_vm, st); + } +} + +void WispThread::park(long millis, const ObjectWaiter* ow) { + assert(ow->_thread->is_Wisp_thread(), "not wisp thread"); + JavaThread* jt = ((WispThread*) ow->_thread)->thread(); + assert(jt == Thread::current(), "current"); + + if (ow->_timeout > 0) { + millis = ow->_timeout; + assert(!ow->_using_wisp_park, "invariant"); + } + + if (ow->_using_wisp_park) { + assert(parkMethod != NULL, "parkMethod should be resolved in set_wisp_booted"); + + ThreadStateTransition::transition(jt, _thread_blocked, _thread_in_vm); + + JavaValue result(T_VOID); + JavaCallArguments args; + args.push_long(millis * 1000000); // to nanos + + // thread steal support + WispPostStealHandleUpdateMark w(jt); // special one, because park() is inside an EnableStealMark, so the _enable_steal_count counter has been added one. + + // when TenantThreadStop is on, we may receive TenantDeathException + // and return before unpark(). + // Do not use a TenantShutdownMark to change the behavior. + JavaCalls::call(&result, methodHandle(Thread::current(), parkMethod), &args, jt); + + // the runtime can not handle the exception on monitorenter bci + // we need clear it to prevent jvm crash + if (jt->has_pending_exception()) { + jt->clear_pending_exception(); + } + + ThreadStateTransition::transition(jt, _thread_in_vm, _thread_blocked); + } else { + if (millis <= 0) { + ow->_event->park(); + } else { + ow->_event->park(millis); + } + } +} + +void WispThread::unpark(int task_id, bool using_wisp_park, bool proxy_unpark, ParkEvent* event, TRAPS) { + if (!using_wisp_park) { + event->unpark(); + return; + } + + JavaThread* jt = THREAD->is_Wisp_thread() ? ((WispThread*) THREAD)->thread() : (JavaThread*) THREAD; + + assert(UseWispMonitor, "UseWispMonitor must be true here"); + bool proxy_unpark_special_case = false; + WispThread* wt = WispThread::current(THREAD); + proxy_unpark_special_case = wt->is_proxy_unpark(); + wt->clear_proxy_unpark_flag(); + + if (proxy_unpark || proxy_unpark_special_case || + jt->is_Compiler_thread() || + jt->is_hidden_from_external_view() || + // SurrogateLockerThread and ServiceThread are "is_hidden_from_external_view()" + jt->is_jvmti_agent_thread()) { // not normal java therad + // proxy_unpark mechanism: + // After we change SystemDictionary_lock from Mutex* to objectMonitor, + // we need to face the reality that some non-JavaThread may produce + // wisp unpark, but they can not invoke java methods. + // So we introduce a unpark dispatch java thread, then append the unpark requests + // to a list, the dispatch thread will fetch and dispatch unparks. + + // due to the fact that we modify the priority of Wisp_lock from `non-leaf` to `special`, + // so we'd use `MutexLocker` and `_no_safepoint_check_flag` to make our program run + MutexLocker mu(Wisp_lock, Mutex::_no_safepoint_check_flag); + _proxy_unpark->append(task_id); + Wisp_lock->notify(); // only one consumer + return; + } + + JavaValue result(T_VOID); + JavaCallArguments args; + args.push_int(task_id); + bool in_java = false; + oop pending_excep = NULL; + const char* pending_file = NULL; + int pending_line = 0; + // Class not found exception may appear here. + // If there exists an exception, the java call could not succeed. + // So must record and clear excpetions here. + if (jt->has_pending_exception()) { + pending_excep = jt->pending_exception(); + pending_file = jt->exception_file(); + pending_line = jt->exception_line(); + jt->clear_pending_exception(); + } + if (jt->thread_state() == _thread_in_Java) { + in_java = true; + ThreadStateTransition::transition_from_java(jt, _thread_in_vm); + } + // Do java calls to perform unpark work. + { + // unpark is very important, should not interruted by tenant shutdown + assert(unparkMethod != NULL, "unparkMethod should be resolved in set_wisp_booted"); + JavaCalls::call(&result, methodHandle(Thread::current(), unparkMethod), &args, jt); + } + // ~tsm may produce an exception and c1 monitor_exit has an exception_mark + // clear the exception to prevent jvm crash + if (jt->has_pending_exception()) { + jt->clear_pending_exception(); + } + if (in_java) { + ThreadStateTransition::transition(jt, _thread_in_vm, _thread_in_Java); + } + if (pending_excep != NULL) { + jt->set_pending_exception(pending_excep, pending_file, pending_line); + } +} + +int WispThread::get_proxy_unpark(jintArray res) { + MutexLocker mu(Wisp_lock, Mutex::_no_safepoint_check_flag); + while (_proxy_unpark == NULL || _proxy_unpark->is_empty()) { + Wisp_lock->wait(); + } + typeArrayOop a = typeArrayOop(JNIHandles::resolve_non_null(res)); + if (a == NULL) { + return 0; + } + int copy_cnt = a->length() < _proxy_unpark->length() ? a->length() : _proxy_unpark->length(); + int copy_start = _proxy_unpark->length() - copy_cnt < 0 ? 0 : _proxy_unpark->length() - copy_cnt; + memcpy(a->int_at_addr(0), _proxy_unpark->adr_at(copy_start), copy_cnt * sizeof(int)); + _proxy_unpark->trunc_to(copy_start); + + return copy_cnt; +} + + +bool WispThread::is_interrupted(bool clear_interrupted) { + if (_coroutine->wisp_task() == NULL) { + // wisp is not initialized + return _thread->is_interrupted(clear_interrupted); + } + int interrupted = com_alibaba_wisp_engine_WispTask::get_interrupted(_coroutine->wisp_task()); + if (interrupted && clear_interrupted) { + com_alibaba_wisp_engine_WispTask::set_interrupted(_coroutine->wisp_task(), 0); + } + return interrupted != 0; +} + +void WispThread::interrupt(int task_id, TRAPS) { + assert(UseWispMonitor, "sanity"); + assert(((JavaThread*) THREAD)->thread_state() == _thread_in_vm, "thread state"); + JavaValue result(T_VOID); + JavaCallArguments args; + args.push_int(task_id); + JavaCalls::call_static(&result, + vmClasses::com_alibaba_wisp_engine_WispTask_klass(), + vmSymbols::interruptById_name(), + vmSymbols::int_void_signature(), + &args, + THREAD); + +} + +EnableStealMark::EnableStealMark(Thread* thread) { + assert(JavaThread::current() == thread, "current thread check"); + if (EnableSteal) { + if (thread->is_Java_thread()) { + _thread = (JavaThread*) thread; + _coroutine = _thread->current_coroutine(); + _enable_steal_count = _coroutine->add_enable_steal_count(); + } else { + _coroutine = NULL; + } + } +} + +EnableStealMark::~EnableStealMark() { + if (EnableSteal && _coroutine != NULL) { + assert(_coroutine->thread() == JavaThread::current(), "must be"); + bool eq = _enable_steal_count == _coroutine->dec_enable_steal_count(); + assert(eq, "enable_steal_count not balanced"); + } +} + +static uintx chunk_wisp_post_steal_handles_do(Chunk *chunk, char *chunk_top, JavaThread *real_thread) { + WispPostStealResource *bottom = (WispPostStealResource *) chunk->bottom(); + WispPostStealResource *top = (WispPostStealResource *) chunk_top; + uintx handles_visited = top - bottom; + assert(top >= bottom && top <= (WispPostStealResource *) chunk->top(), "just checking"); + while (bottom < top) { + // we overwrite our real thread or jnienv to every slot + if (bottom->type == WispPostStealResource::ThreadRef) { + bottom->update_thread_ref(real_thread); + } else if (bottom->type == WispPostStealResource::JNIEnvRef) { + bottom->update_jnienv_ref(real_thread->jni_environment()); + } else { + ShouldNotReachHere(); + } + bottom++; + } + return handles_visited; +} + +void WispResourceArea::wisp_post_steal_handles_do(JavaThread *real_thread) { + uintx handles_visited = 0; + // First handle the current chunk. It is filled to the high water mark. + handles_visited += chunk_wisp_post_steal_handles_do(_chunk, _hwm, real_thread); + // Then handle all previous chunks. They are completely filled. + Chunk *k = _first; + while (k != _chunk) { + handles_visited += chunk_wisp_post_steal_handles_do(k, k->top(), real_thread); + k = k->next(); + } +} + +class WispPostStealHandle : public StackObj { +private: + void allocate_handle(WispResourceArea *area, Thread **t) { + WispPostStealResource *src = area->real_allocate_handle(); + src->type = WispPostStealResource::ThreadRef; + src->u.thread_ref = t; + } + void allocate_handle(WispResourceArea *area, JNIEnv **t) { + WispPostStealResource *src = area->real_allocate_handle(); + src->type = WispPostStealResource::JNIEnvRef; + src->u.jnienv_ref = t; + } +public: + template explicit WispPostStealHandle(T* stackObj) { + Thread *& ref = stackObj->thread_ref(); + JavaThread *thread = ((JavaThread *)ref); + assert(thread->is_Java_thread(), "must be"); + allocate_handle(thread->current_coroutine()->wisp_post_steal_resource_area(), &ref); + } + explicit WispPostStealHandle(Thread ** thread_ptr) { + JavaThread *thread = (JavaThread *)*thread_ptr; + assert((*thread_ptr)->is_Java_thread(), "must be"); + allocate_handle(thread->current_coroutine()->wisp_post_steal_resource_area(), thread_ptr); + } + explicit WispPostStealHandle(JNIEnv ** jnienv_ptr) { + JavaThread *thread = JavaThread::thread_from_jni_environment(*jnienv_ptr); + assert(thread->is_Java_thread(), "must be"); + allocate_handle(thread->current_coroutine()->wisp_post_steal_resource_area(), jnienv_ptr); + } +}; + +WispPostStealHandleUpdateMark::WispPostStealHandleUpdateMark(JavaThread *& th1, Thread *& th2, + methodHandle & m1, methodHandle *& m2, + JavaCallWrapper & w) +{ + initialize(th1); + + if (!_success) return; + WispPostStealHandle h((Thread **)&th1); + WispPostStealHandle h1(&th2); + WispPostStealHandle h2(&m1); + WispPostStealHandle h3(m2); + WispPostStealHandle h4(&w); +} + +WispPostStealHandleUpdateMark::WispPostStealHandleUpdateMark(Thread *& th, + methodHandle *m1, methodHandle *m2, + JavaCallWrapper *w) +{ + initialize(*((JavaThread **)&th)); + + if (!_success) return; + WispPostStealHandle h(&th); + if (m1) { WispPostStealHandle h(m1); } + if (m2) { WispPostStealHandle h(m2); } + if (w) { WispPostStealHandle h(w); } +} + +WispPostStealHandleUpdateMark::WispPostStealHandleUpdateMark(JavaThread *& th1, Thread *& th2, + JNIEnv *& env, ThreadInVMfromNative & tiv, HandleMarkCleaner & hmc, + JavaThreadInObjectWaitState *jtios, vframeStream *f, methodHandle *m) +{ + initialize(th1); + + if (!_success) return; + WispPostStealHandle h((Thread **)&th1); + WispPostStealHandle h1(&th2); + WispPostStealHandle h2(&env); + WispPostStealHandle h3(&tiv); + WispPostStealHandle h4(&hmc); + if (jtios) { WispPostStealHandle h(jtios); } + if (f) { WispPostStealHandle h(f); } + if (m) { WispPostStealHandle h(m); } +} + +WispPostStealHandleUpdateMark::WispPostStealHandleUpdateMark(JavaThread *& th1, Thread *& th2, + ThreadInVMfromJava & tiva, HandleMarkCleaner & hmc) +{ + initialize(th1); + + if (!_success) return; + WispPostStealHandle h((Thread **)&th1); + WispPostStealHandle h1(&th2); + WispPostStealHandle h2(&tiva); + WispPostStealHandle h3(&hmc); +} + +WispPostStealHandleUpdateMark::WispPostStealHandleUpdateMark(JavaThread *& th1, Thread *& th2, + ThreadInVMfromJava & tiva) +{ + initialize(th1); + + if (!_success) return; + WispPostStealHandle h((Thread **)&th1); + WispPostStealHandle h1(&th2); + WispPostStealHandle h2(&tiva); +} + +WispPostStealHandleUpdateMark::WispPostStealHandleUpdateMark(HandleMarkCleaner & hmc) +{ + assert(hmc.thread_ref()->is_Java_thread(), "sanity"); + initialize((JavaThread*)hmc.thread_ref()); + + if (!_success) return; + WispPostStealHandle h(&hmc); +} + +WispPostStealHandleUpdateMark::WispPostStealHandleUpdateMark(JavaThread *thread, ThreadBlockInVM & tbv) +{ + initialize(thread, true); + + if (!_success) return; + WispPostStealHandle h(&(tbv._tbivmpp)); +} + +WispPostStealHandleUpdateMark::WispPostStealHandleUpdateMark(JavaThread *&th) // this is a special one +{ + initialize(th, true); + + if (!_success) return; + WispPostStealHandle h((Thread **)&th); +} + +bool WispPostStealHandleUpdateMark::check(JavaThread *current, bool sp) { + assert(current == JavaThread::current(), "must be"); + Coroutine *coroutine = current->current_coroutine(); + if (!EnableSteal || + coroutine == current->coroutine_list() || + // if we won't steal it, then no need to allocate handles. + coroutine->enable_steal_count() != (sp ? current->java_call_counter()+1 : current->java_call_counter())) { + _success = false; + return false; + } + _success = true; + return true; +} + +WispPostStealHandleUpdateMark::~WispPostStealHandleUpdateMark() { + if (!_success) return; + WispResourceArea* area = _area; // help compilers with poor alias analysis + assert(area == _coroutine->wisp_post_steal_resource_area(), "sanity check"); + assert(area->_nesting > 0, "must stack allocate HandleMarks" ); + debug_only(area->_nesting--); + + // Delete later chunks + if( _chunk->next() ) { + // reset arena size before delete chunks. Otherwise, the total + // arena size could exceed total chunk size + assert(area->size_in_bytes() > size_in_bytes(), "Sanity check"); + area->set_size_in_bytes(size_in_bytes()); + _chunk->next_chop(); + } else { + assert(area->size_in_bytes() == size_in_bytes(), "Sanity check"); + } + // Roll back arena to saved top markers + area->_chunk = _chunk; + area->_hwm = _hwm; + area->_max = _max; +#ifdef ASSERT + // clear out first chunk (to detect allocation bugs) + if (ZapVMHandleArea) { + memset(_hwm, badHandleValue, _max - _hwm); + } +#endif + +} + +void Coroutine::initialize_coroutine_support(JavaThread* thread) { + assert(EnableCoroutine, "Coroutine is disabled"); + guarantee(thread == JavaThread::current(), "sanity check"); + + if (UseWispMonitor) { + assert(thread->parker(), "sanity check"); + java_lang_Thread::set_park_event( + thread->threadObj(), (uintptr_t) thread->parker()); + } + + EXCEPTION_MARK + HandleMark hm(thread); + Handle obj(thread, thread->threadObj()); + JavaValue result(T_VOID); + + JavaCalls::call_virtual(&result, + obj, + vmClasses::Thread_klass(), + vmSymbols::initializeCoroutineSupport_method_name(), + vmSymbols::void_method_signature(), + thread); +} + diff --git a/src/hotspot/share/runtime/coroutine.hpp b/src/hotspot/share/runtime/coroutine.hpp index a7b8d4e5875..6bbbe1418ed 100644 --- a/src/hotspot/share/runtime/coroutine.hpp +++ b/src/hotspot/share/runtime/coroutine.hpp @@ -25,6 +25,8 @@ #ifndef SHARE_VM_RUNTIME_COROUTINE_HPP #define SHARE_VM_RUNTIME_COROUTINE_HPP +#include "classfile/javaThreadStatus.hpp" +#include "classfile/vmSymbols.hpp" #include "runtime/jniHandles.hpp" #include "runtime/handles.hpp" #include "memory/allocation.hpp" @@ -32,7 +34,8 @@ #include "memory/virtualspace.hpp" #include "runtime/javaFrameAnchor.hpp" #include "runtime/monitorChunk.hpp" - +#include "runtime/osThread.hpp" +#include "runtime/thread.inline.hpp" // number of heap words that prepareSwitch will add as a safety measure to the CoroutineData size #define COROUTINE_DATA_OVERSIZE (64) @@ -48,6 +51,7 @@ class Coroutine; class CoroutineStack; +class WispThread; template @@ -76,6 +80,8 @@ class FrameClosure: public StackObj { virtual void frames_do(frame* fr, RegisterMap* map) = 0; }; +class WispPostStealHandleUpdateMark; +class WispResourceArea; class Coroutine: public CHeapObj, public DoublyLinkedList { public: @@ -86,6 +92,9 @@ class Coroutine: public CHeapObj, public DoublyLinkedList { _dead = 0x00000003, // TODO is this really needed? _dummy = 0xffffffff }; + enum Consts { + WISP_ID_NOT_SET = -1, + }; private: CoroutineState _state; @@ -95,13 +104,25 @@ class Coroutine: public CHeapObj, public DoublyLinkedList { JavaThread* _thread; CoroutineStack* _stack; + int _wisp_task_id; + oop _wisp_engine; + oop _wisp_task; + WispThread* _wisp_thread; + ResourceArea* _resource_area; HandleArea* _handle_area; HandleMark* _last_handle_mark; JNIHandleBlock* _active_handles; GrowableArray* _metadata_handles; + JavaFrameAnchor _anchor; + // PrivilegedElement* _privileged_stack_top; + JavaThreadStatus _thread_status; + int _enable_steal_count; int _java_call_counter; + // work steal pool + WispResourceArea* _wisp_post_steal_resource_area; + #ifdef _LP64 intptr_t _storage[2]; #endif @@ -116,6 +137,7 @@ class Coroutine: public CHeapObj, public DoublyLinkedList { void run(jobject coroutine); + static void initialize_coroutine_support(JavaThread* thread); static Coroutine* create_thread_coroutine(JavaThread* thread, CoroutineStack* stack); static Coroutine* create_coroutine(JavaThread* thread, CoroutineStack* stack, oop coroutineObj); @@ -129,9 +151,24 @@ class Coroutine: public CHeapObj, public DoublyLinkedList { CoroutineStack* stack() const { return _stack; } + int wisp_task_id() const { return _wisp_task_id; } + void set_wisp_task_id(int x) { _wisp_task_id = x; } + + oop wisp_engine() const { return _wisp_engine; } + void set_wisp_engine(oop x); + + oop wisp_task() const { return _wisp_task; } + void set_wisp_task(oop x) { _wisp_task = x; } + + WispThread* wisp_thread() const { return _wisp_thread; } + ResourceArea* resource_area() const { return _resource_area; } void set_resource_area(ResourceArea* x) { _resource_area = x; } + // work steal support + WispResourceArea* wisp_post_steal_resource_area() const { return _wisp_post_steal_resource_area; } + void change_thread_for_wisp(JavaThread *thread); + HandleArea* handle_area() const { return _handle_area; } void set_handle_area(HandleArea* x) { _handle_area = x; } @@ -144,17 +181,25 @@ class Coroutine: public CHeapObj, public DoublyLinkedList { GrowableArray* metadata_handles() const { return _metadata_handles; } void set_metadata_handles(GrowableArray* handles){ _metadata_handles = handles; } + int enable_steal_count() { return _enable_steal_count; } + int add_enable_steal_count() { return ++_enable_steal_count; } + int dec_enable_steal_count() { return _enable_steal_count--; } + int java_call_counter() const { return _java_call_counter; } void set_java_call_counter(int x) { _java_call_counter = x; } bool is_disposable(); + void print_stack_on(outputStream* st); + // GC support void oops_do(OopClosure* f, CodeBlobClosure* cf); void nmethods_do(CodeBlobClosure* cf); void metadata_do(MetadataClosure* f); void frames_do(void f(frame*, const RegisterMap* map)); + static ByteSize thread_offset() { return byte_offset_of(Coroutine, _thread); } + static ByteSize state_offset() { return byte_offset_of(Coroutine, _state); } static ByteSize stack_offset() { return byte_offset_of(Coroutine, _stack); } @@ -163,9 +208,15 @@ class Coroutine: public CHeapObj, public DoublyLinkedList { static ByteSize last_handle_mark_offset() { return byte_offset_of(Coroutine, _last_handle_mark); } static ByteSize active_handles_offset() { return byte_offset_of(Coroutine, _active_handles); } static ByteSize metadata_handles_offset() { return byte_offset_of(Coroutine, _metadata_handles); } -#ifdef ASSERT + // static ByteSize privileged_stack_top_offset(){ return byte_offset_of(Coroutine, _privileged_stack_top); } + static ByteSize last_Java_sp_offset() { + return byte_offset_of(Coroutine, _anchor) + JavaFrameAnchor::last_Java_sp_offset(); + } + static ByteSize last_Java_pc_offset() { + return byte_offset_of(Coroutine, _anchor) + JavaFrameAnchor::last_Java_pc_offset(); + } + static ByteSize thread_status_offset() { return byte_offset_of(Coroutine, _thread_status); } static ByteSize java_call_counter_offset() { return byte_offset_of(Coroutine, _java_call_counter); } -#endif #ifdef _LP64 static ByteSize storage_offset() { return byte_offset_of(Coroutine, _storage); } @@ -177,6 +228,7 @@ class Coroutine: public CHeapObj, public DoublyLinkedList { public: static ByteSize last_SEH_offset() { return byte_offset_of(Coroutine, _last_SEH); } #endif + static ByteSize wisp_thread_offset() { return byte_offset_of(Coroutine, _wisp_thread); } }; class CoroutineStack: public CHeapObj, public DoublyLinkedList { @@ -250,4 +302,237 @@ template void DoublyLinkedList::insert_into_list(pointer& list) { } } +class ObjectWaiter; + +class WispThread: public JavaThread { + friend class Coroutine; +private: + static bool _wisp_booted; + static Method* parkMethod; + static Method* unparkMethod; + static GrowableArray* _proxy_unpark; + + Coroutine* _coroutine; + JavaThread* _thread; + + bool _is_proxy_unpark; + + WispThread(Coroutine *c): _coroutine(c), _thread(c->thread()), _is_proxy_unpark(false) { + set_osthread(new OSThread(NULL, NULL)); + set_thread_state(_thread_in_vm); + } + + virtual ~WispThread() { + delete osthread(); + set_osthread(NULL); + } + +public: + static bool wisp_booted() { return _wisp_booted; } + static void set_wisp_booted(JavaThread* thread); + + virtual bool is_Wisp_thread() const { return true; } + + Coroutine* coroutine() const { return _coroutine; } + + JavaThread* thread() const { return _thread; } + + void change_thread(JavaThread *thread) { assert(thread->is_Java_thread(), "must be"); _thread = thread; } // steal support + + bool is_interrupted(bool clear_interrupted); + static void interrupt(int task_id, TRAPS); + + static ByteSize thread_offset() { return byte_offset_of(WispThread, _thread); } + + bool is_proxy_unpark() { return _is_proxy_unpark; } + void set_proxy_unpark_flag() { _is_proxy_unpark = true; } + void clear_proxy_unpark_flag() { _is_proxy_unpark = false; } + + virtual bool is_lock_owned(address adr) const { + CoroutineStack* stack = _coroutine->stack(); + return stack->stack_base() >= adr && + adr > (stack->stack_base() - stack->stack_size()); + } + + // we must set ObjectWaiter members before node enqueued(observed by other threads) + void before_enqueue(ObjectMonitor* monitor, ObjectWaiter* ow); + static void park(long millis, const ObjectWaiter* ow); + static void unpark(const ObjectWaiter* ow, TRAPS) { + unpark(ow->_park_wisp_id, ow->_using_wisp_park, ow->_proxy_wisp_unpark, ow->_event, THREAD); + } + static void unpark(int task_id, bool using_wisp_park, bool proxy_unpark, ParkEvent* event, TRAPS); + static int get_proxy_unpark(jintArray res); + + static WispThread* current(Thread* thread) { + assert(thread->is_Java_thread(), "invariant") ; + return thread->is_Wisp_thread() ? (WispThread*) thread : + ((JavaThread*) thread)->current_coroutine()->wisp_thread(); + } +}; + +// we supported coroutine stealing for following native calls: +// 1. sun.reflect.NativeMethodAccessorImpl.invoke0() +// 2. sun.reflect.NativeConstructorAccessorImpl.newInstance0() +// 3. java.security.AccessController.doPrivileged() +// 4. java.lang.Object.wait() +// 5,6,7. monitorenter for interp, c1 and c2 +class EnableStealMark: public StackObj { +private: + JavaThread* _thread; + Coroutine* _coroutine; + int _enable_steal_count; +public: + EnableStealMark(Thread* thread); + ~EnableStealMark(); +}; + +struct WispStealCandidate { +private: + Symbol *_holder; + Symbol *_name; + Symbol *_signature; +public: + WispStealCandidate (Symbol *holder, Symbol *name, Symbol *signature) : _holder(holder), _name(name), _signature(signature) {} +private: + bool is_method_invoke() { + return _holder == vmSymbols::sun_reflect_NativeMethodAccessorImpl() && // sun.reflect.NativeMethodAccessorImpl.invoke0() + _name == vmSymbols::invoke0_name() && + _signature == vmSymbols::invoke0_signature(); + } + bool is_constructor_newinstance() { + return _holder == vmSymbols::sun_reflect_NativeConstructorAccessorImpl() && // sun.reflect.NativeConstructorAccessorImpl.newInstance0() + _name == vmSymbols::newInstance0_name() && + _signature == vmSymbols::newInstance0_signature(); + } + bool is_doPrivilege() { + return _holder == vmSymbols::java_security_AccessController() && // java.security.AccessController.doPrivilege(***) + _name == vmSymbols::doPrivileged_name() && + (_signature == vmSymbols::doPrivileged_signature_1() || // doPrivilege() has four native functions + _signature == vmSymbols::doPrivileged_signature_2() || + _signature == vmSymbols::doPrivileged_signature_3() || + _signature == vmSymbols::doPrivileged_signature_4()); + } + bool is_object_wait() { + return _holder == vmSymbols::java_lang_Object() && // java.lang.Object.wait() + _name == vmSymbols::wait_name() && + _signature == vmSymbols::long_void_signature(); + } +public: + bool is_steal_candidate() { + return is_constructor_newinstance() || is_doPrivilege() || is_method_invoke() || is_object_wait(); + } +}; + +struct WispPostStealResource { +public: + union { + Thread **thread_ref; + JNIEnv **jnienv_ref; + } u; + enum Type{ + ThreadRef, + JNIEnvRef + } type; +public: + void update_thread_ref(JavaThread *real_thread) { *u.thread_ref = real_thread; } + void update_jnienv_ref(JNIEnv *real_jnienv) { *u.jnienv_ref = real_jnienv; } +}; + +// First letter indicates type of the frame: +// J: Java frame (compiled) +// j: Java frame (interpreted) +// V: VM frame (C/C++) +// v: Other frames running VM generated code (e.g. stubs, adapters, etc.) +// C: C/C++ frame +#define WISP_THREAD_UPDATE get_thread(r15) +#define WISP_X86_CONVENTION_V2J_UPDATE __ WISP_THREAD_UPDATE +#define WISP_X86_CONVENTION_V2j_UPDATE __ WISP_THREAD_UPDATE +#define WISP_COMPILER_RESTORE_FORCE_UPDATE __ WISP_THREAD_UPDATE +#define WISP_j2v_UPDATE __ movptr(thread, r15_thread) +#define WISP_V2v_UPDATE WISP_THREAD_UPDATE + +class WispPostStealHandleUpdateMark; + +class WispResourceArea: public ResourceArea { + friend class WispPostStealHandleUpdateMark; +private: + Coroutine *_outer; +public: + WispResourceArea(Coroutine *outer, MEMFLAGS flags = mtWisp) : ResourceArea(flags), _outer(outer) {} + + WispResourceArea(Coroutine *outer, size_t init_size, MEMFLAGS flags = mtWisp) : ResourceArea(init_size, flags), _outer(outer) { } + + WispPostStealResource* real_allocate_handle() { +#ifdef ASSERT + WispPostStealResource *src = (WispPostStealResource *) (UseMallocOnly ? internal_malloc_4(sizeof(WispPostStealResource)) : Amalloc_4(sizeof(WispPostStealResource))); +#else + WispPostStealResource *src = (WispPostStealResource *) Amalloc_4(sizeof(WispPostStealResource)); +#endif + return src; + } + + // thread steal support + void wisp_post_steal_handles_do(JavaThread *real_thread); +}; + +inline void Coroutine::change_thread_for_wisp(JavaThread *thread) { + this->wisp_post_steal_resource_area()->wisp_post_steal_handles_do(thread); +} + +class ThreadInVMfromNative; +class ThreadInVMfromJava; +class HandleMarkCleaner; +class JavaThreadInObjectWaitState; +class ThreadBlockInVM; +class vframeStream; + +class WispPostStealHandleUpdateMark: public StackObj { +private: + WispResourceArea *_area; // Resource area to stack allocate + Chunk *_chunk; // saved arena chunk + char *_hwm, *_max; + size_t _size_in_bytes; + Coroutine *_coroutine; + bool _success; + + bool check(JavaThread *current, bool sp = false); + + void initialize(JavaThread *thread, bool sp = false) { + if (!check(thread, sp)) return; + Coroutine *coroutine = thread->current_coroutine(); + _coroutine = coroutine; + _area = coroutine->wisp_post_steal_resource_area(); + _chunk = _area->_chunk; + _hwm = _area->_hwm; + _max= _area->_max; + _size_in_bytes = _area->size_in_bytes(); + debug_only(_area->_nesting++;) + assert( _area->_nesting > 0, "must stack allocate RMs" ); + } +public: + WispPostStealHandleUpdateMark(JavaThread *& th1, Thread *& th2, // constructor only for JavaCalls::call() + methodHandle & m1, methodHandle *& m2, + JavaCallWrapper & w); + WispPostStealHandleUpdateMark(Thread *& th, // constructor for misc + methodHandle *m1 = NULL, methodHandle *m2 = NULL, + JavaCallWrapper *w = NULL); + WispPostStealHandleUpdateMark(JavaThread *& th1, Thread *& th2, // constructor only for JVM_ENTRY + JNIEnv *& env, ThreadInVMfromNative & tiv, HandleMarkCleaner & hmc, + JavaThreadInObjectWaitState *jtios = NULL, vframeStream *f = NULL, methodHandle *m = NULL); + WispPostStealHandleUpdateMark(JavaThread *& th1, Thread *& th2, // constructor only for Interpreter::monitorenter + ThreadInVMfromJava & tiva, HandleMarkCleaner & hmc); + WispPostStealHandleUpdateMark(JavaThread *& th1, Thread *& th2, // constructor for other monitorenters + ThreadInVMfromJava & tiva); + WispPostStealHandleUpdateMark(HandleMarkCleaner & hmc); + WispPostStealHandleUpdateMark(JavaThread *thread, ThreadBlockInVM & tbv); // constructor is used inside objectMonitor call, which is also within EnableStealMark scope. + WispPostStealHandleUpdateMark(JavaThread *& th); // this is a special one, used for a fix inside EnableStealMark + + ~WispPostStealHandleUpdateMark(); + + +private: + size_t size_in_bytes() { return _size_in_bytes; } +}; + + #endif // SHARE_VM_RUNTIME_COROUTINE_HPP diff --git a/src/hotspot/share/runtime/globals.hpp b/src/hotspot/share/runtime/globals.hpp index aeddd3d4636..2ab57a3ecf0 100644 --- a/src/hotspot/share/runtime/globals.hpp +++ b/src/hotspot/share/runtime/globals.hpp @@ -2097,6 +2097,19 @@ const intx ObjectAlignmentInBytes = 8; \ product(uintx, MaxFreeCoroutinesCacheSize, 20, \ "The max number of free coroutine stacks a thread can keep") \ + \ + product(bool, EnableCoroutineTimeSlice, false, \ + "Wisp will perform yield periodically") \ + \ + product(bool, UseWispMonitor, false, \ + "yields to next coroutine when ObjectMonitor is contended") \ + \ + product(bool, EnableSteal, true, \ + "Enable coroutine steal") \ + \ + product(bool, UseWisp2, false, \ + "Enable Wisp2") \ + \ // end of RUNTIME_FLAGS diff --git a/src/hotspot/share/runtime/handles.hpp b/src/hotspot/share/runtime/handles.hpp index 40c5bda6816..b8ecd42c7fa 100644 --- a/src/hotspot/share/runtime/handles.hpp +++ b/src/hotspot/share/runtime/handles.hpp @@ -154,6 +154,8 @@ DEF_HANDLE(typeArray , is_typeArray_noinline ) ~name##Handle (); \ void remove(); \ \ + Thread *& thread_ref() { return _thread; } \ + \ /* Operators for ease of use */ \ type* operator () () const { return obj(); } \ type* operator -> () const { return non_null_obj(); } \ @@ -278,6 +280,9 @@ class HandleMark { void* operator new [](size_t size) throw(); void operator delete(void* p); void operator delete[](void* p); + + // only for wisp + void change_thread_for_wisp(Thread *thread); }; //------------------------------------------------------------------------------------------------------------------------ @@ -322,6 +327,10 @@ class HandleMarkCleaner: public StackObj { public: inline HandleMarkCleaner(Thread* thread); inline ~HandleMarkCleaner(); + Thread *& thread_ref() { + assert(EnableCoroutine, "EnableCoroutine is off"); + return _thread; + } }; #endif // SHARE_RUNTIME_HANDLES_HPP diff --git a/src/hotspot/share/runtime/handles.inline.hpp b/src/hotspot/share/runtime/handles.inline.hpp index 491ede99884..11110a0430a 100644 --- a/src/hotspot/share/runtime/handles.inline.hpp +++ b/src/hotspot/share/runtime/handles.inline.hpp @@ -90,6 +90,17 @@ inline void HandleMark::pop_and_restore() { debug_only(_area->_handle_mark_nesting--); } +inline void HandleMark::change_thread_for_wisp(Thread *thread) { + assert(EnableCoroutine, "Coroutine is disabled"); + if (_thread == thread) return; + HandleMark *hm = this; + // change thread for the whole list + while (hm != NULL) { + hm->_thread = thread; + hm = hm->_previous_handle_mark; + } +} + inline HandleMarkCleaner::HandleMarkCleaner(Thread* thread) { _thread = thread; _thread->last_handle_mark()->push(); diff --git a/src/hotspot/share/runtime/interfaceSupport.inline.hpp b/src/hotspot/share/runtime/interfaceSupport.inline.hpp index 7cb6eeff078..24475c46d8e 100644 --- a/src/hotspot/share/runtime/interfaceSupport.inline.hpp +++ b/src/hotspot/share/runtime/interfaceSupport.inline.hpp @@ -29,6 +29,7 @@ // No interfaceSupport.hpp #include "gc/shared/gc_globals.hpp" +#include "runtime/coroutine.hpp" #include "runtime/handles.inline.hpp" #include "runtime/mutexLocker.hpp" #include "runtime/orderAccess.hpp" @@ -86,6 +87,9 @@ class ThreadStateTransition : public StackObj { // Change threadstate in a manner, so safepoint can detect changes. // Time-critical: called on exit from every runtime routine static inline void transition(JavaThread *thread, JavaThreadState from, JavaThreadState to) { + if (UseWispMonitor && thread->is_Wisp_thread()) { + thread = ((WispThread*)thread)->thread(); + } assert(from != _thread_in_Java, "use transition_from_java"); assert(from != _thread_in_native, "use transition_from_native"); assert((from & 1) == 0 && (to & 1) == 0, "odd numbers are transitions states"); @@ -125,6 +129,11 @@ class ThreadStateTransition : public StackObj { thread->set_thread_state(to); } + Thread *& thread_ref() { + assert(EnableCoroutine, "EnableCoroutine is off"); + return (Thread *&)_thread; + } + protected: void trans(JavaThreadState from, JavaThreadState to) { transition(_thread, from, to); } void trans_from_java(JavaThreadState to) { transition_from_java(_thread, to); } @@ -287,6 +296,8 @@ class InFlightMutexRelease { // access and release said mutex when transitioning back from blocked to vm (destructor) in // case we need to stop for a safepoint or handshake. class ThreadBlockInVM { + friend class WispPostStealHandleUpdateMark; + InFlightMutexRelease _ifmr; ThreadBlockInVMPreprocess _tbivmpp; public: diff --git a/src/hotspot/share/runtime/javaCalls.cpp b/src/hotspot/share/runtime/javaCalls.cpp index a624396c946..e78249924bc 100644 --- a/src/hotspot/share/runtime/javaCalls.cpp +++ b/src/hotspot/share/runtime/javaCalls.cpp @@ -99,7 +99,7 @@ JavaCallWrapper::JavaCallWrapper(const methodHandle& callee_method, Handle recei _anchor.copy(_thread->frame_anchor()); _thread->frame_anchor()->clear(); - debug_only(_thread->inc_java_call_counter()); + _thread->inc_java_call_counter(); _thread->set_active_handles(new_handles); // install new handle block and reset Java frame linkage assert (_thread->thread_state() != _thread_in_native, "cannot set native pc to NULL"); @@ -133,7 +133,7 @@ JavaCallWrapper::~JavaCallWrapper() { _thread->frame_anchor()->zap(); - debug_only(_thread->dec_java_call_counter()); + _thread->dec_java_call_counter(); // Old thread-local info. has been restored. We are not back in the VM. ThreadStateTransition::transition_from_java(_thread, _thread_in_vm); @@ -347,6 +347,9 @@ Handle JavaCalls::construct_new_instance(InstanceKlass* klass, Symbol* construct void JavaCalls::call(JavaValue* result, const methodHandle& method, JavaCallArguments* args, TRAPS) { // Check if we need to wrap a potential OS exception handler around thread. // This is used for e.g. Win32 structured exception handlers. + assert(!UseWispMonitor || !is_init_completed() || + java_lang_Thread::park_event(((JavaThread*) THREAD)->threadObj()), + "park_event need to be set before calling java"); // Need to wrap each and every time, since there might be native code down the // stack that has installed its own exception handlers. os::os_exception_wrapper(call_helper, result, method, args, THREAD); @@ -435,6 +438,11 @@ void JavaCalls::call_helper(JavaValue* result, const methodHandle& method, JavaC } } #endif + // thread steal support + methodHandle* m = const_cast(&method); + Thread* t = THREAD; + WispPostStealHandleUpdateMark w(thread, t, *m, m, link); + StubRoutines::call_stub()( (address)&link, // (intptr_t*)&(result->_value), // see NOTE above (compiler problem) diff --git a/src/hotspot/share/runtime/javaCalls.hpp b/src/hotspot/share/runtime/javaCalls.hpp index 20ba2fd7ca7..4c2de35fc53 100644 --- a/src/hotspot/share/runtime/javaCalls.hpp +++ b/src/hotspot/share/runtime/javaCalls.hpp @@ -72,6 +72,10 @@ class JavaCallWrapper: StackObj { bool is_first_frame() const { return _anchor.last_Java_sp() == NULL; } + Thread *& thread_ref() { + assert(EnableCoroutine, "EnableCoroutine is off"); + return (Thread *&)_thread; + } }; diff --git a/src/hotspot/share/runtime/mutexLocker.cpp b/src/hotspot/share/runtime/mutexLocker.cpp index cb3e01f6e79..d9a9fc8bea0 100644 --- a/src/hotspot/share/runtime/mutexLocker.cpp +++ b/src/hotspot/share/runtime/mutexLocker.cpp @@ -41,7 +41,7 @@ Mutex* Patching_lock = NULL; Mutex* CompiledMethod_lock = NULL; -Monitor* SystemDictionary_lock = NULL; +Monitor* SystemDictionary_monitor_lock= NULL; Mutex* SharedDictionary_lock = NULL; Monitor* ClassInitError_lock = NULL; Mutex* Module_lock = NULL; @@ -164,6 +164,10 @@ Monitor* JVMCI_lock = NULL; #endif +SystemDictMonitor* SystemDictionary_lock = NULL; + +Monitor* Wisp_lock = NULL; + #define MAX_NUM_MUTEX 128 static Mutex* _mutex_array[MAX_NUM_MUTEX]; static int _num_mutex; @@ -198,6 +202,40 @@ void assert_locked_or_safepoint_or_handshake(const Mutex* lock, const JavaThread if (thread->is_handshake_safe_for(Thread::current())) return; assert_locked_or_safepoint(lock); } + +static bool is_owner(const SystemDictMonitor* lock, Thread* THREAD) { + if (lock->is_obj_lock()) { + assert(UseWispMonitor, "should UseWispMonitor"); + WispThread* wt = WispThread::current(Thread::current()); + if (ObjectSynchronizer::current_thread_holds_lock(wt, Handle(Thread::current(), lock->obj()))) { + return true; + } + } else if (lock->monitor()->owner() == THREAD) { + return true; + } + return false; +} + +void assert_lock_strong(const SystemDictMonitor* lock) { + if (IgnoreLockingAssertions) return; + assert(lock != NULL, "Need non-NULL lock"); + if (is_owner(lock, Thread::current())) return; + fatal("must own lock %s", lock->monitor()->name()); +} + +void assert_locked_or_safepoint(const SystemDictMonitor* lock) { + // check if this thread owns the lock (common case) + if (IgnoreLockingAssertions) return; + assert(lock != NULL, "Need non-NULL lock"); + if (SafepointSynchronize::is_at_safepoint()) return; + if (is_owner(lock, Thread::current())) return; + if (!Universe::is_fully_initialized()) return; + // see if invoker of VM operation owns it + VM_Operation* op = VMThread::vm_operation(); + if (op != NULL && is_owner(lock, op->calling_thread())) return; + fatal("must own lock %s", lock->monitor()->name()); +} + #endif #define def(var, type, pri, vm_block, safepoint_check_allowed ) { \ @@ -254,7 +292,7 @@ void mutex_init() { def(JmethodIdCreation_lock , PaddedMutex , special-2, true, _safepoint_check_never); // used for creating jmethodIDs. - def(SystemDictionary_lock , PaddedMonitor, leaf, true, _safepoint_check_always); + def(SystemDictionary_monitor_lock, PaddedMonitor, leaf, true, _safepoint_check_always); def(SharedDictionary_lock , PaddedMutex , leaf, true, _safepoint_check_always); def(ClassInitError_lock , PaddedMonitor, leaf+1, true, _safepoint_check_always); def(Module_lock , PaddedMutex , leaf+2, false, _safepoint_check_always); @@ -334,6 +372,7 @@ void mutex_init() { def(NMethodSweeperStats_lock , PaddedMutex , special, true, _safepoint_check_never); def(ThreadsSMRDelete_lock , PaddedMonitor, special, true, _safepoint_check_never); def(ThreadIdTableCreate_lock , PaddedMutex , leaf, false, _safepoint_check_always); + def(Wisp_lock , PaddedMonitor, special, true, _safepoint_check_never); def(SharedDecoder_lock , PaddedMutex , native, true, _safepoint_check_never); def(DCmdFactory_lock , PaddedMutex , leaf, true, _safepoint_check_never); #if INCLUDE_NMT @@ -354,6 +393,9 @@ void mutex_init() { #if INCLUDE_JVMCI def(JVMCI_lock , PaddedMonitor, nonleaf+2, true, _safepoint_check_always); #endif + SystemDictionary_lock = UseWispMonitor ? + new SystemDictObjMonitor(SystemDictionary_monitor_lock): + new SystemDictMonitor(SystemDictionary_monitor_lock); } GCMutexLocker::GCMutexLocker(Mutex* mutex) { @@ -366,6 +408,9 @@ GCMutexLocker::GCMutexLocker(Mutex* mutex) { } } +GCSystemDictLocker::GCSystemDictLocker(SystemDictMonitor* mutex) + : SystemDictLocker(JavaThread::current(), mutex, !SafepointSynchronize::is_at_safepoint()) {} + // Print all mutexes/monitors that are currently owned by a thread; called // by fatal error handler. void print_owned_locks_on_error(outputStream* st) { diff --git a/src/hotspot/share/runtime/mutexLocker.hpp b/src/hotspot/share/runtime/mutexLocker.hpp index 90bf00a1b13..0893d940abe 100644 --- a/src/hotspot/share/runtime/mutexLocker.hpp +++ b/src/hotspot/share/runtime/mutexLocker.hpp @@ -33,7 +33,8 @@ extern Mutex* Patching_lock; // a lock used to guard code patching of compiled code extern Mutex* CompiledMethod_lock; // a lock used to guard a compiled method and OSR queues -extern Monitor* SystemDictionary_lock; // a lock on the system dictionary +extern Monitor* SystemDictionary_monitor_lock; // a lock on the system dictonary +extern SystemDictMonitor* SystemDictionary_lock; // a lock on the system dictonary, transform to ObjectMonitor after wisp is booted extern Mutex* SharedDictionary_lock; // a lock on the CDS shared dictionary extern Monitor* ClassInitError_lock; // a lock on the class initialization error table extern Mutex* Module_lock; // a lock on module and package related data structures @@ -162,6 +163,7 @@ extern Mutex* Bootclasspath_lock; extern Mutex* tty_lock; // lock to synchronize output. +extern Monitor* Wisp_lock; // used to sync Wisp operations // A MutexLocker provides mutual exclusion with respect to a given mutex // for the scope which contains the locker. The lock is an OS lock, not // an object lock, and the two do not interoperate. Do not use Mutex-based @@ -188,6 +190,10 @@ void assert_locked_or_safepoint(const Mutex* lock); void assert_locked_or_safepoint_weak(const Mutex* lock); void assert_lock_strong(const Mutex* lock); void assert_locked_or_safepoint_or_handshake(const Mutex* lock, const JavaThread* thread); +void assert_locked_or_safepoint(const SystemDictMonitor * lock); +void assert_locked_or_safepoint_weak(const SystemDictMonitor * lock); +void assert_lock_strong(const SystemDictMonitor * lock); +void assert_locked_or_safepoint_or_handshake(const SystemDictMonitor * lock, const JavaThread* thread); #else #define assert_locked_or_safepoint(lock) #define assert_locked_or_safepoint_weak(lock) @@ -314,4 +320,34 @@ class MutexUnlocker: StackObj { } }; +class SystemDictLocker: StackObj { + private: + SystemDictMonitor* _mutex; + JavaThread* _thread; + BasicLock _lock; + bool _locked; + public: + SystemDictLocker(JavaThread* THREAD, SystemDictMonitor* mutex, bool do_lock=true) { + _locked = do_lock; + _mutex = mutex; + _thread = THREAD; + if (do_lock) { + _mutex->lock(&_lock, _thread); + } + } + + void wait() { _mutex->wait(&_lock, _thread); } + void notify_all() { _mutex->notify_all(_thread); } + void lock() { _mutex->lock(&_lock, _thread); } + void unlock() { _mutex->unlock(&_lock, _thread); } + + + ~SystemDictLocker() { if (_locked) _mutex->unlock(&_lock, _thread); } +}; + +class GCSystemDictLocker: public SystemDictLocker { + public: + GCSystemDictLocker(SystemDictMonitor* mutex); +}; + #endif // SHARE_RUNTIME_MUTEXLOCKER_HPP diff --git a/src/hotspot/share/runtime/objectMonitor.cpp b/src/hotspot/share/runtime/objectMonitor.cpp index a9faccd9e20..e404201725b 100644 --- a/src/hotspot/share/runtime/objectMonitor.cpp +++ b/src/hotspot/share/runtime/objectMonitor.cpp @@ -50,7 +50,9 @@ #include "runtime/safefetch.inline.hpp" #include "runtime/safepointMechanism.inline.hpp" #include "runtime/sharedRuntime.hpp" +#include "runtime/thread.hpp" #include "runtime/thread.inline.hpp" +#include "runtime/coroutine.hpp" #include "services/threadService.hpp" #include "utilities/dtrace.hpp" #include "utilities/macros.hpp" @@ -321,6 +323,9 @@ void ObjectMonitor::ClearSuccOnSuspend::operator()(JavaThread* current) { bool ObjectMonitor::enter(JavaThread* current) { // The following code is ordered to check the most common cases first // and to reduce RTS->RTO cache line upgrades on SPARC and IA32 processors. + if (UseWispMonitor) { + current = WispThread::current(current); + } void* cur = try_set_owner_from(NULL, current); if (cur == NULL) { @@ -342,7 +347,7 @@ bool ObjectMonitor::enter(JavaThread* current) { } // We've encountered genuine contention. - assert(current->_Stalled == 0, "invariant"); + assert(current->_Stalled == 0 || UseWispMonitor, "invariant"); current->_Stalled = intptr_t(this); // Try one round of spinning *before* enqueueing current @@ -483,6 +488,9 @@ bool ObjectMonitor::enter(JavaThread* current) { // Callers must compensate as needed. int ObjectMonitor::TryLock(JavaThread* current) { + if (UseWispMonitor) { + current = WispThread::current(current); + } void* own = owner_raw(); if (own != NULL) return 0; if (try_set_owner_from(NULL, current) == NULL) { @@ -678,6 +686,9 @@ const char* ObjectMonitor::is_busy_to_string(stringStream* ss) { #define MAX_RECHECK_INTERVAL 1000 void ObjectMonitor::EnterI(JavaThread* current) { + if (UseWispMonitor) { + current = WispThread::current(current); + } assert(current->thread_state() == _thread_blocked, "invariant"); // Try the lock - TATAS @@ -740,6 +751,9 @@ void ObjectMonitor::EnterI(JavaThread* current) { current->_ParkEvent->reset(); node._prev = (ObjectWaiter*) 0xBAD; node.TState = ObjectWaiter::TS_CXQ; + if (UseWispMonitor) { + ((WispThread*) current)->before_enqueue(this, &node); + } // Push "current" onto the front of the _cxq. // Once on cxq/EntryList, current stays on-queue until it acquires the lock. @@ -810,14 +824,22 @@ void ObjectMonitor::EnterI(JavaThread* current) { // park self if (_Responsible == current) { - current->_ParkEvent->park((jlong) recheckInterval); + if (UseWispMonitor) { + WispThread::park(recheckInterval, &node); + } else { + current->_ParkEvent->park((jlong) recheckInterval); + } // Increase the recheckInterval, but clamp the value. recheckInterval *= 8; if (recheckInterval > MAX_RECHECK_INTERVAL) { recheckInterval = MAX_RECHECK_INTERVAL; } } else { - current->_ParkEvent->park(); + if (UseWispMonitor) { + WispThread::park(-1, &node); + } else { + current->_ParkEvent->park(); + } } if (TryLock(current) > 0) break; @@ -938,6 +960,9 @@ void ObjectMonitor::EnterI(JavaThread* current) { // In the future we should reconcile EnterI() and ReenterI(). void ObjectMonitor::ReenterI(JavaThread* current, ObjectWaiter* currentNode) { + if (UseWispMonitor) { + current = WispThread::current(current); + } assert(current != NULL, "invariant"); assert(currentNode != NULL, "invariant"); assert(currentNode->_thread == current, "invariant"); @@ -963,7 +988,11 @@ void ObjectMonitor::ReenterI(JavaThread* current, ObjectWaiter* currentNode) { { ClearSuccOnSuspend csos(this); ThreadBlockInVMPreprocess tbivs(current, csos); - current->_ParkEvent->park(); + if (UseWispMonitor) { + WispThread::park(MAX_RECHECK_INTERVAL, currentNode); + } else { + current->_ParkEvent->park(); + } } } @@ -1014,6 +1043,9 @@ void ObjectMonitor::ReenterI(JavaThread* current, ObjectWaiter* currentNode) { // unlinking the thread until ::exit()-time. void ObjectMonitor::UnlinkAfterAcquire(JavaThread* current, ObjectWaiter* currentNode) { + if (UseWispMonitor) { + current = WispThread::current(current); + } assert(owner_raw() == current, "invariant"); assert(currentNode->_thread == current, "invariant"); @@ -1133,6 +1165,9 @@ void ObjectMonitor::UnlinkAfterAcquire(JavaThread* current, ObjectWaiter* curren // of such futile wakups is low. void ObjectMonitor::exit(JavaThread* current, bool not_suspended) { + if (UseWispMonitor) { + current = WispThread::current(current); + } void* cur = owner_raw(); if (current != cur) { if (current->is_lock_owned((address)cur)) { @@ -1312,6 +1347,9 @@ void ObjectMonitor::exit(JavaThread* current, bool not_suspended) { } void ObjectMonitor::ExitEpilog(JavaThread* current, ObjectWaiter* Wakee) { + if (UseWispMonitor) { + current = WispThread::current(current); + } assert(owner_raw() == current, "invariant"); // Exit protocol: @@ -1322,6 +1360,10 @@ void ObjectMonitor::ExitEpilog(JavaThread* current, ObjectWaiter* Wakee) { _succ = Wakee->_thread; ParkEvent * Trigger = Wakee->_event; + const int wisp_id = Wakee->_park_wisp_id; + const bool use_wisp = Wakee->_using_wisp_park; + const bool proxy_unpark = Wakee->_proxy_wisp_unpark; + // Hygiene -- once we've set _owner = NULL we can't safely dereference Wakee again. // The thread associated with Wakee may have grabbed the lock and "Wakee" may be @@ -1334,7 +1376,11 @@ void ObjectMonitor::ExitEpilog(JavaThread* current, ObjectWaiter* Wakee) { OrderAccess::fence(); DTRACE_MONITOR_PROBE(contended__exit, this, object(), current); - Trigger->unpark(); + if (UseWispMonitor) { + WispThread::unpark(wisp_id, use_wisp, proxy_unpark, Trigger, current); + } else { + Trigger->unpark(); + } // Maintain stats and report events to JVMTI OM_PERFDATA_OP(Parks, inc()); @@ -1351,6 +1397,9 @@ void ObjectMonitor::ExitEpilog(JavaThread* current, ObjectWaiter* Wakee) { // inflated monitor, e.g. the monitor can be inflated by a non-owning // thread due to contention. intx ObjectMonitor::complete_exit(JavaThread* current) { + if (UseWispMonitor) { + current = WispThread::current(current); + } assert(InitDone, "Unexpectedly not initialized"); void* cur = owner_raw(); @@ -1402,6 +1451,9 @@ bool ObjectMonitor::reenter(intx recursions, JavaThread* current) { // (IMSE). If there is a pending exception and the specified thread // is not the owner, that exception will be replaced by the IMSE. bool ObjectMonitor::check_owner(TRAPS) { + if (UseWispMonitor) { + THREAD = WispThread::current(THREAD); + } JavaThread* current = THREAD; void* cur = owner_raw(); if (cur == current) { @@ -1444,12 +1496,24 @@ static void post_monitor_wait_event(EventJavaMonitorWait* event, event->commit(); } +static bool check_interrupt(Thread* self, bool clear_interrupted) { + if (UseWispMonitor) { + assert(self->is_Wisp_thread(), "must be"); + return ((WispThread*) self)->is_interrupted(clear_interrupted); + } else { + return ((JavaThread*) self)->is_interrupted(clear_interrupted); + } +} + // ----------------------------------------------------------------------------- // Wait/Notify/NotifyAll // // Note: a subset of changes to ObjectMonitor::wait() // will need to be replicated in complete_exit void ObjectMonitor::wait(jlong millis, bool interruptible, TRAPS) { + if (UseWispMonitor) { + THREAD = WispThread::current(THREAD); + } JavaThread* current = THREAD; assert(InitDone, "Unexpectedly not initialized"); @@ -1459,7 +1523,7 @@ void ObjectMonitor::wait(jlong millis, bool interruptible, TRAPS) { EventJavaMonitorWait event; // check for a pending interrupt - if (interruptible && current->is_interrupted(true) && !HAS_PENDING_EXCEPTION) { + if (interruptible && check_interrupt(current, true) && !HAS_PENDING_EXCEPTION) { // post monitor waited event. Note that this is past-tense, we are done waiting. if (JvmtiExport::should_post_monitor_waited()) { // Note: 'false' parameter is passed here because the @@ -1481,7 +1545,7 @@ void ObjectMonitor::wait(jlong millis, bool interruptible, TRAPS) { return; } - assert(current->_Stalled == 0, "invariant"); + assert(current->_Stalled || UseWispMonitor == 0, "invariant"); current->_Stalled = intptr_t(this); current->set_current_waiting_monitor(this); @@ -1491,6 +1555,10 @@ void ObjectMonitor::wait(jlong millis, bool interruptible, TRAPS) { ObjectWaiter node(current); node.TState = ObjectWaiter::TS_WAIT; current->_ParkEvent->reset(); + if (UseWispMonitor) { + ((WispThread*) current)->before_enqueue(this, &node); + } + OrderAccess::fence(); // ST into Event; membar ; LD interrupted-flag // Enter the waiting queue, which is a circular doubly linked list in this case @@ -1537,10 +1605,14 @@ void ObjectMonitor::wait(jlong millis, bool interruptible, TRAPS) { if (interrupted || HAS_PENDING_EXCEPTION) { // Intentionally empty } else if (node._notified == 0) { - if (millis <= 0) { - current->_ParkEvent->park(); + if (UseWispMonitor) { + WispThread::park(millis, &node); } else { - ret = current->_ParkEvent->park(millis); + if (millis <= 0) { + current->_ParkEvent->park(); + } else { + ret = current->_ParkEvent->park(millis); + } } } } @@ -1606,7 +1678,11 @@ void ObjectMonitor::wait(jlong millis, bool interruptible, TRAPS) { // We redo the unpark() to ensure forward progress, i.e., we // don't want all pending threads hanging (parked) with none // entering the unlocked monitor. - node._event->unpark(); + if (UseWispMonitor) { + WispThread::unpark(&node, THREAD); + } else { + node._event->unpark(); + } } } @@ -1616,7 +1692,7 @@ void ObjectMonitor::wait(jlong millis, bool interruptible, TRAPS) { OrderAccess::fence(); - assert(current->_Stalled != 0, "invariant"); + assert(current->_Stalled != 0 || UseWispMonitor, "invariant"); current->_Stalled = 0; assert(owner_raw() != current, "invariant"); @@ -1654,7 +1730,7 @@ void ObjectMonitor::wait(jlong millis, bool interruptible, TRAPS) { if (!WasNotified) { // no, it could be timeout or Thread.interrupt() or both // check for interrupt event, otherwise it is timeout - if (interruptible && current->is_interrupted(true) && !HAS_PENDING_EXCEPTION) { + if (interruptible && check_interrupt(current, true) && !HAS_PENDING_EXCEPTION) { THROW(vmSymbols::java_lang_InterruptedException()); } } @@ -1670,6 +1746,9 @@ void ObjectMonitor::wait(jlong millis, bool interruptible, TRAPS) { // we might just dequeue a thread from the WaitSet and directly unpark() it. void ObjectMonitor::INotify(JavaThread* current) { + if (UseWispMonitor) { + current = WispThread::current(current); + } Thread::SpinAcquire(&_WaitSetLock, "WaitSet - notify"); ObjectWaiter* iterator = DequeueWaiter(); if (iterator != NULL) { @@ -1732,6 +1811,9 @@ void ObjectMonitor::INotify(JavaThread* current) { // that suggests a lost wakeup bug. void ObjectMonitor::notify(TRAPS) { + if (UseWispMonitor) { + THREAD = WispThread::current(THREAD); + } JavaThread* current = THREAD; CHECK_OWNER(); // Throws IMSE if not owner. if (_WaitSet == NULL) { @@ -1751,6 +1833,9 @@ void ObjectMonitor::notify(TRAPS) { // mode the waitset will be empty and the EntryList will be "DCBAXYZ". void ObjectMonitor::notifyAll(TRAPS) { + if (UseWispMonitor) { + THREAD = WispThread::current(THREAD); + } JavaThread* current = THREAD; CHECK_OWNER(); // Throws IMSE if not owner. if (_WaitSet == NULL) { @@ -1835,6 +1920,9 @@ void ObjectMonitor::notifyAll(TRAPS) { // Spinning: Fixed frequency (100%), vary duration int ObjectMonitor::TrySpin(JavaThread* current) { + if (UseWispMonitor) { + current = WispThread::current(current); + } // Dumb, brutal spin. Good for comparative measurements against adaptive spinning. int ctr = Knob_FixedSpin; if (ctr != 0) { @@ -2070,6 +2158,11 @@ ObjectWaiter::ObjectWaiter(JavaThread* current) { TState = TS_RUN; _thread = current; _event = _thread->_ParkEvent; + if (UseWispMonitor && current->is_Wisp_thread()) { + // JvmtiRawMonitor::SimpleWait use this class directly. + // In this scenario, JavaThread* is passed even UseWispMonitor enabled + _event = ((WispThread*)current)->thread()->_ParkEvent ; + } _active = false; assert(_event != NULL, "invariant"); } diff --git a/src/hotspot/share/runtime/objectMonitor.hpp b/src/hotspot/share/runtime/objectMonitor.hpp index da3a4b09827..0ae2df949c0 100644 --- a/src/hotspot/share/runtime/objectMonitor.hpp +++ b/src/hotspot/share/runtime/objectMonitor.hpp @@ -52,6 +52,11 @@ class ObjectWaiter : public StackObj { volatile int _notified; volatile TStates TState; bool _active; // Contention monitoring is enabled + int _park_wisp_id; + bool _using_wisp_park; + bool _proxy_wisp_unpark; + long _timeout; + public: ObjectWaiter(JavaThread* current); diff --git a/src/hotspot/share/runtime/objectMonitor.inline.hpp b/src/hotspot/share/runtime/objectMonitor.inline.hpp index 09389425733..cfdf3683db1 100644 --- a/src/hotspot/share/runtime/objectMonitor.inline.hpp +++ b/src/hotspot/share/runtime/objectMonitor.inline.hpp @@ -30,9 +30,13 @@ #include "logging/log.hpp" #include "oops/access.inline.hpp" #include "runtime/atomic.hpp" +#include "runtime/coroutine.hpp" #include "runtime/synchronizer.hpp" inline intptr_t ObjectMonitor::is_entered(JavaThread* current) const { + if (UseWispMonitor) { + current = WispThread::current(current); + } void* owner = owner_raw(); if (current == owner || current->is_lock_owned((address)owner)) { return 1; diff --git a/src/hotspot/share/runtime/os.cpp b/src/hotspot/share/runtime/os.cpp index ef43c317040..fc41ef21461 100644 --- a/src/hotspot/share/runtime/os.cpp +++ b/src/hotspot/share/runtime/os.cpp @@ -448,6 +448,10 @@ static void signal_thread_entry(JavaThread* thread, TRAPS) { } } +bool os::is_signal_dispatcher_thread(JavaThread* thread) { + return thread->is_expected_thread_entry(&signal_thread_entry); +} + void os::init_before_ergo() { initialize_initial_active_processor_count(); // We need to initialize large page support here because ergonomics takes some diff --git a/src/hotspot/share/runtime/os.hpp b/src/hotspot/share/runtime/os.hpp index 9e074a1d023..5bb451c9e25 100644 --- a/src/hotspot/share/runtime/os.hpp +++ b/src/hotspot/share/runtime/os.hpp @@ -483,6 +483,8 @@ class os: AllStatic { static bool signal_thread(Thread* thread, int sig, const char* reason); static void free_thread(OSThread* osthread); + + static bool is_signal_dispatcher_thread(JavaThread* thread); // thread id on Linux/64bit is 64bit, on Windows it's 32bit static intx current_thread_id(); @@ -1037,4 +1039,6 @@ class os: AllStatic { extern "C" int SpinPause(); +bool clear_interrupt_for_wisp(Thread *); + #endif // SHARE_RUNTIME_OS_HPP diff --git a/src/hotspot/share/runtime/reflection.cpp b/src/hotspot/share/runtime/reflection.cpp index a7e232124b1..5ef52d50450 100644 --- a/src/hotspot/share/runtime/reflection.cpp +++ b/src/hotspot/share/runtime/reflection.cpp @@ -43,6 +43,7 @@ #include "oops/oop.inline.hpp" #include "oops/typeArrayOop.inline.hpp" #include "prims/jvmtiExport.hpp" +#include "runtime/coroutine.hpp" #include "runtime/fieldDescriptor.inline.hpp" #include "runtime/handles.inline.hpp" #include "runtime/javaCalls.hpp" @@ -1105,7 +1106,13 @@ static oop invoke(InstanceKlass* klass, // All oops (including receiver) is passed in as Handles. An potential oop is returned as an // oop (i.e., NOT as an handle) JavaValue result(rtype); - JavaCalls::call(&result, method, &java_args, THREAD); + + { + Thread* t = THREAD; + WispPostStealHandleUpdateMark w(t, const_cast(&reflected_method), &method); + EnableStealMark p(THREAD); + JavaCalls::call(&result, method, &java_args, THREAD); + } if (HAS_PENDING_EXCEPTION) { // Method threw an exception; wrap it in an InvocationTargetException @@ -1151,6 +1158,10 @@ oop Reflection::invoke_method(oop method_mirror, Handle receiver, objArrayHandle } methodHandle method(THREAD, m); + // Coroutine work steal support + Thread* t = THREAD; + WispPostStealHandleUpdateMark w(t, &method); + return invoke(klass, method, receiver, override, ptypes, rtype, args, true, THREAD); } @@ -1176,6 +1187,10 @@ oop Reflection::invoke_constructor(oop constructor_mirror, objArrayHandle args, klass->check_valid_for_instantiation(false, CHECK_NULL); Handle receiver = klass->allocate_instance_handle(CHECK_NULL); + // Coroutine work steal support + Thread* t = THREAD; + WispPostStealHandleUpdateMark w(t, &method); + // Ignore result from call and return receiver invoke(klass, method, receiver, override, ptypes, T_VOID, args, false, CHECK_NULL); return receiver(); diff --git a/src/hotspot/share/runtime/sharedRuntime.cpp b/src/hotspot/share/runtime/sharedRuntime.cpp index 66613e5d8c6..c609321ac94 100644 --- a/src/hotspot/share/runtime/sharedRuntime.cpp +++ b/src/hotspot/share/runtime/sharedRuntime.cpp @@ -256,6 +256,27 @@ JRT_LEAF(jfloat, SharedRuntime::frem(jfloat x, jfloat y)) #endif JRT_END +JRT_ENTRY_NO_ASYNC(int, SharedRuntime::wisp_yield(JavaThread* current, Method* method)) + assert(EnableCoroutine, "Coroutine is disabled"); + JavaThread* jt = current; + JavaValue result(T_VOID); + JavaCallArguments args; + // Class not found exception may appear here. + // We do not continue yield with exceptions. + if (jt->has_pending_exception()) { + return 0; + } + jt->set_wisp_preempt(0); + assert(jt->thread_state() == _thread_in_vm, "sanity check"); + // Do java calls to perform yield work. + JavaCalls::call_static(&result, + vmClasses::Thread_klass(), + vmSymbols::yield_name(), + vmSymbols::void_method_signature(), + &args, + jt); + return 0; +JRT_END JRT_LEAF(jdouble, SharedRuntime::drem(jdouble x, jdouble y)) #ifdef _WIN64 @@ -2128,6 +2149,14 @@ void SharedRuntime::monitor_enter_helper(oopDesc* obj, BasicLock* lock, JavaThre if (PrintBiasedLockingStatistics) { Atomic::inc(BiasedLocking::slow_path_entry_count_addr()); } + + // must place it befor EnableStealMark. + Thread* t = THREAD; + WispPostStealHandleUpdateMark w(current, t, __tiv); + + // Coroutine work steal support + EnableStealMark p(THREAD); + Handle h_obj(THREAD, obj); ObjectSynchronizer::enter(h_obj, lock, current); assert(!HAS_PENDING_EXCEPTION, "Should have no exception here"); @@ -2136,9 +2165,46 @@ void SharedRuntime::monitor_enter_helper(oopDesc* obj, BasicLock* lock, JavaThre // Handles the uncommon case in locking, i.e., contention or an inflated lock. JRT_BLOCK_ENTRY(void, SharedRuntime::complete_monitor_locking_C(oopDesc* obj, BasicLock* lock, JavaThread* current)) + WispPostStealHandleUpdateMark w(__hm); SharedRuntime::monitor_enter_helper(obj, lock, current); JRT_END +JRT_ENTRY_NO_ASYNC(void, SharedRuntime::complete_wisp_monitor_unlocking_C(oopDesc* obj, BasicLock* lock, JavaThread* current)) + assert(EnableCoroutine, "Coroutine is disabled"); + JavaThread* cur_thread = JavaThread::current(); + // I'm not convinced we need the code contained by MIGHT_HAVE_PENDING anymore + // testing was unable to ever fire the assert that guarded it so I have removed it. + assert(!HAS_PENDING_EXCEPTION, "Do we need code below anymore?"); +#undef MIGHT_HAVE_PENDING +#ifdef MIGHT_HAVE_PENDING + // Save and restore any pending_exception around the exception mark. + // While the slow_exit must not throw an exception, we could come into + // this routine with one set. + oop pending_excep = NULL; + const char* pending_file; + int pending_line; + if (HAS_PENDING_EXCEPTION) { + pending_excep = PENDING_EXCEPTION; + pending_file = cur_thread->exception_file(); + pending_line = cur_thread->exception_line(); + CLEAR_PENDING_EXCEPTION; + } +#endif /* MIGHT_HAVE_PENDING */ + + // Use handle to access objects since we will call java code + Handle h_obj(current, obj); + { + EXCEPTION_MARK; + ObjectSynchronizer::exit(h_obj, lock, cur_thread); + } + +#ifdef MIGHT_HAVE_PENDING + if (pending_excep != NULL) { + cur_thread->set_pending_exception(pending_excep, pending_file, pending_line); + } +#endif /* MIGHT_HAVE_PENDING */ +JRT_END + void SharedRuntime::monitor_exit_helper(oopDesc* obj, BasicLock* lock, JavaThread* current) { assert(JavaThread::current() == current, "invariant"); // Exit must be non-blocking, and therefore no exceptions can be thrown. @@ -2151,7 +2217,13 @@ void SharedRuntime::monitor_exit_helper(oopDesc* obj, BasicLock* lock, JavaThrea } return; } - ObjectSynchronizer::exit(obj, lock, current); + if (UseWispMonitor) { + HandleMarkCleaner __hm(current); + Handle h_obj(current, obj); + ObjectSynchronizer::exit(h_obj, lock, current); + } else { + ObjectSynchronizer::exit(obj, lock, current); + } } // Handles the uncommon cases of monitor unlocking in compiled code diff --git a/src/hotspot/share/runtime/sharedRuntime.hpp b/src/hotspot/share/runtime/sharedRuntime.hpp index 8912a7e8bd0..f2967a89a75 100644 --- a/src/hotspot/share/runtime/sharedRuntime.hpp +++ b/src/hotspot/share/runtime/sharedRuntime.hpp @@ -275,6 +275,7 @@ class SharedRuntime: AllStatic { static int dtrace_object_alloc_base(Thread* thread, oopDesc* o, int size); static int dtrace_method_entry(JavaThread* thread, Method* m); static int dtrace_method_exit(JavaThread* thread, Method* m); + static int wisp_yield(JavaThread* thread, Method* m); // Utility method for retrieving the Java thread id, returns 0 if the // thread is not a well formed Java thread. @@ -491,6 +492,7 @@ class SharedRuntime: AllStatic { // Slow-path Locking and Unlocking static void complete_monitor_locking_C(oopDesc* obj, BasicLock* lock, JavaThread* current); static void complete_monitor_unlocking_C(oopDesc* obj, BasicLock* lock, JavaThread* current); + static void complete_wisp_monitor_unlocking_C(oopDesc* obj, BasicLock* lock, JavaThread* current); // Resolving of calls static address resolve_static_call_C (JavaThread* current); diff --git a/src/hotspot/share/runtime/synchronizer.cpp b/src/hotspot/share/runtime/synchronizer.cpp index b0e0059b16b..c9bab5b59d2 100644 --- a/src/hotspot/share/runtime/synchronizer.cpp +++ b/src/hotspot/share/runtime/synchronizer.cpp @@ -35,6 +35,7 @@ #include "oops/oop.inline.hpp" #include "runtime/atomic.hpp" #include "runtime/biasedLocking.hpp" +#include "runtime/coroutine.hpp" #include "runtime/handles.inline.hpp" #include "runtime/handshake.hpp" #include "runtime/interfaceSupport.inline.hpp" @@ -265,6 +266,9 @@ static uintx _no_progress_cnt = 0; // into a single notifyAndExit() runtime primitive. bool ObjectSynchronizer::quick_notify(oopDesc* obj, JavaThread* current, bool all) { + if (UseWispMonitor) { + current = WispThread::current(current); + } assert(current->thread_state() == _thread_in_Java, "invariant"); NoSafepointVerifier nsv; if (obj == NULL) return false; // slow-path for invalid obj @@ -313,7 +317,10 @@ bool ObjectSynchronizer::quick_notify(oopDesc* obj, JavaThread* current, bool al bool ObjectSynchronizer::quick_enter(oop obj, JavaThread* current, BasicLock * lock) { - assert(current->thread_state() == _thread_in_Java, "invariant"); + if (UseWispMonitor) { + current = WispThread::current(current); + } + assert(UseWispMonitor || current->thread_state() == _thread_in_Java, "invariant"); NoSafepointVerifier nsv; if (obj == NULL) return false; // Need to throw NPE @@ -420,6 +427,57 @@ void ObjectSynchronizer::handle_sync_on_value_based_class(Handle obj, JavaThread } } +void ObjectSynchronizer::exit(Handle object, BasicLock* lock, TRAPS) { + assert(EnableCoroutine, "Coroutine is disabled"); + markWord mark = object->mark(); + // We cannot check for Biased Locking if we are racing an inflation. + assert(mark == markWord::INFLATING() || + !mark->has_bias_pattern(), "should not see bias pattern here"); + + markWord dhw = lock->displaced_header(); + if (dhw.value() == 0) { + // If the displaced header is NULL, then this exit matches up with + // a recursive enter. No real work to do here except for diagnostics. +#ifndef PRODUCT + if (mark != markWord::INFLATING()) { + // Only do diagnostics if we are not racing an inflation. Simply + // exiting a recursive enter of a Java Monitor that is being + // inflated is safe; see the has_monitor() comment below. + assert(!mark->is_neutral(), "invariant"); + assert(!mark->has_locker() || + THREAD->is_lock_owned((address)mark->locker()), "invariant"); + if (mark->has_monitor()) { + // The BasicLock's displaced_header is marked as a recursive + // enter and we have an inflated Java Monitor (ObjectMonitor). + // This is a special case where the Java Monitor was inflated + // after this thread entered the stack-lock recursively. When a + // Java Monitor is inflated, we cannot safely walk the Java + // Monitor owner's stack and update the BasicLocks because a + // Java Monitor can be asynchronously inflated by a thread that + // does not own the Java Monitor. + ObjectMonitor * m = mark->monitor(); + assert(((oop)(m->object()))->mark() == mark, "invariant"); + assert(m->is_entered(THREAD), "invariant"); + } + } +#endif + return; + } + + if (mark == markWord::from_pointer(lock)) { + // If the object is stack-locked by the current thread, try to + // swing the displaced header from the BasicLock back to the mark. + assert(dhw->is_neutral(), "invariant"); + if (object->cas_set_mark(dhw, mark) == mark) { + return; + } + } + + // We have to take the slow-path of possible inflation and then exit. + ObjectSynchronizer::inflate(THREAD, + object(), + inflate_cause_vm_internal)->exit(THREAD, true); +} // ----------------------------------------------------------------------------- // Monitor Enter/Exit // The interpreter and compiler assembly code tries to lock using the fast path @@ -623,6 +681,7 @@ ObjectLocker::ObjectLocker(Handle obj, JavaThread* thread) { if (_obj() != NULL) { ObjectSynchronizer::enter(_obj, &_lock, _thread); + assert(!EnableSteal || thread == Thread::current(), "must not be stealed"); } } @@ -990,7 +1049,9 @@ bool ObjectSynchronizer::current_thread_holds_lock(JavaThread* current, assert(!h_obj->mark().has_bias_pattern(), "biases should be revoked by now"); } - assert(current == JavaThread::current(), "Can only be called on current thread"); + assert(current == JavaThread::current() || + UseWispMonitor && current == WispThread::current(JavaThread::current()), + "Can only be called on current thread"); oop obj = h_obj(); markWord mark = read_stable_mark(obj); @@ -1572,7 +1633,7 @@ class ReleaseJavaMonitorsClosure: public MonitorClosure { void ObjectSynchronizer::release_monitors_owned_by_thread(JavaThread* current) { assert(current == JavaThread::current(), "must be current Java thread"); NoSafepointVerifier nsv; - ReleaseJavaMonitorsClosure rjmc(current); + ReleaseJavaMonitorsClosure rjmc(UseWispMonitor ? WispThread::current(current) : current); ObjectSynchronizer::monitors_iterate(&rjmc, current); assert(!current->has_pending_exception(), "Should not be possible"); current->clear_pending_exception(); @@ -1784,3 +1845,58 @@ void ObjectSynchronizer::log_in_use_monitor_details(outputStream* out) { out->flush(); } + +void SystemDictObjMonitor::lock(BasicLock* lock, TRAPS) { + assert(UseWispMonitor, "SystemDictObjMonitor if only for UseWispMonitor"); + if (!is_obj_lock()) { + _monitor->lock(); + if (is_obj_lock()) { + // set_obj_lock has been called before we fetch the lock. + // now _monitor is Pointless. re-fetch the objectMonitor to + // satisfy the critical section semantics. + _monitor->unlock(); + ObjectSynchronizer::enter(Handle(THREAD, _obj), lock, THREAD); + } + } else { + ObjectSynchronizer::enter(Handle(THREAD, _obj), lock, THREAD); + } +} + +void SystemDictObjMonitor::unlock(BasicLock* lock, TRAPS) { + assert(UseWispMonitor, "SystemDictObjMonitor if only for UseWispMonitor"); + if (!is_obj_lock()) { + _monitor->unlock(); + } else { + ObjectSynchronizer::exit(_obj, lock, THREAD); + } +} + +void SystemDictObjMonitor::wait(BasicLock* lock, TRAPS) { + assert(UseWispMonitor, "SystemDictObjMonitor if only for UseWispMonitor"); + if (!is_obj_lock()) { + _monitor->wait(); + if (is_obj_lock()) { + _monitor->unlock(); + ObjectSynchronizer::enter(Handle(THREAD, _obj), lock, THREAD); + } + } else { + ObjectSynchronizer::wait(Handle(THREAD, _obj), 0, THREAD); + } +} + +void SystemDictObjMonitor::notify_all(TRAPS) { + assert(UseWispMonitor, "SystemDictObjMonitor if only for UseWispMonitor"); + if (!is_obj_lock()) { + _monitor->notify_all(); + } else { + ObjectSynchronizer::notifyall(Handle(THREAD, _obj), THREAD); + } +} + +void SystemDictObjMonitor::set_obj_lock(oop obj, TRAPS) { + MutexLocker mu(THREAD, _monitor); + assert(UseWispMonitor, "UseWispMonitor if off"); + assert(_obj == NULL, "_obj already been set"); + _obj = obj; + _monitor->notify_all(); +} diff --git a/src/hotspot/share/runtime/synchronizer.hpp b/src/hotspot/share/runtime/synchronizer.hpp index 5a331e306ee..fcaa31cb628 100644 --- a/src/hotspot/share/runtime/synchronizer.hpp +++ b/src/hotspot/share/runtime/synchronizer.hpp @@ -29,6 +29,7 @@ #include "oops/markWord.hpp" #include "runtime/basicLock.hpp" #include "runtime/handles.hpp" +#include "runtime/mutex.hpp" #include "utilities/growableArray.hpp" class LogStream; @@ -90,6 +91,7 @@ class ObjectSynchronizer : AllStatic { // This is the "slow path" version of monitor enter and exit. static void enter(Handle obj, BasicLock* lock, JavaThread* current); static void exit(oop obj, BasicLock* lock, JavaThread* current); + static void exit(Handle obj, BasicLock* lock, JavaThread* current); // Used only to handle jni locks or other unmatched monitor enter/exit // Internally they will use heavy weight monitor. @@ -206,4 +208,37 @@ class ObjectLocker : public StackObj { void wait_uninterruptibly(JavaThread* current) { ObjectSynchronizer::wait_uninterruptibly(_obj, current); } }; +class SystemDictMonitor : public CHeapObj { + protected: + Monitor* _monitor; + public: + SystemDictMonitor(Monitor* monitor): _monitor(monitor) { } + virtual void lock(BasicLock* lock, TRAPS) { _monitor->lock(); } + virtual void unlock(BasicLock* lock, TRAPS) { _monitor->unlock(); } + virtual void wait(BasicLock* lock, TRAPS) { _monitor->wait(); } + virtual void notify_all(TRAPS) { _monitor->notify_all(); } + virtual void set_obj_lock(oop obj, TRAPS) { ShouldNotReachHere(); } + Monitor* monitor() const { return _monitor; } + virtual oop obj() const { return NULL; } + virtual bool is_obj_lock() const { return false; } + virtual void oops_do(OopClosure* f) { } +}; + +class SystemDictObjMonitor : public SystemDictMonitor { + private: + oop _obj; + public: + SystemDictObjMonitor(Monitor* monitor): SystemDictMonitor(monitor), _obj(NULL) { + assert(UseWispMonitor, "SystemDictObjMonitor if only for UseWispMonitor"); + } + virtual void lock(BasicLock* lock, TRAPS); + virtual void unlock(BasicLock* lock, TRAPS); + virtual void wait(BasicLock* lock, TRAPS); + virtual void notify_all(TRAPS); + virtual void set_obj_lock(oop obj, TRAPS); + virtual oop obj() const { return _obj; } + virtual bool is_obj_lock() const { return _obj != NULL; } + virtual void oops_do(OopClosure* f) { if (_obj != NULL) f->do_oop(&_obj); } +}; + #endif // SHARE_RUNTIME_SYNCHRONIZER_HPP diff --git a/src/hotspot/share/runtime/thread.cpp b/src/hotspot/share/runtime/thread.cpp index f8eb4c27b50..a04fbeb8d6c 100644 --- a/src/hotspot/share/runtime/thread.cpp +++ b/src/hotspot/share/runtime/thread.cpp @@ -461,6 +461,9 @@ Thread::~Thread() { // A JavaThread is considered dangling if it not handshake-safe with respect to // the current thread, it is not on a ThreadsList, or not at safepoint. void Thread::check_for_dangling_thread_pointer(Thread *thread) { + if (UseWispMonitor && thread->is_Wisp_thread()) { + thread = ((WispThread*) thread)->thread(); + } assert(!thread->is_Java_thread() || thread->as_Java_thread()->is_handshake_safe_for(Thread::current()) || !thread->as_Java_thread()->on_thread_list() || @@ -769,6 +772,22 @@ static void create_initial_thread(Handle thread_group, JavaThread* thread, JavaThreadStatus::RUNNABLE); } +static void call_initializeWispClass(TRAPS) { + assert(EnableCoroutine, "Coroutine is disabled"); + Klass* klass = SystemDictionary::resolve_or_fail(vmSymbols::com_alibaba_wisp_engine_WispEngine(), true, CHECK); + JavaValue result(T_VOID); + JavaCalls::call_static(&result, klass, vmSymbols::initializeWispClass_name(), + vmSymbols::void_method_signature(), CHECK); +} + +static void call_startWispDaemons(TRAPS) { + assert(EnableCoroutine, "Coroutine is disabled"); + Klass* klass = SystemDictionary::resolve_or_fail(vmSymbols::com_alibaba_wisp_engine_WispEngine(), true, CHECK); + JavaValue result(T_VOID); + JavaCalls::call_static(&result, klass, vmSymbols::startWispDaemons_name(), + vmSymbols::void_method_signature(), CHECK); +} + // Extract version and vendor specific information from // java.lang.VersionProps fields. // Returned char* is allocated in the thread's resource area @@ -1026,6 +1045,7 @@ JavaThread::JavaThread() : _callee_target(nullptr), _vm_result(nullptr), _vm_result_2(nullptr), + _vm_result_for_wisp(nullptr), _current_pending_monitor(NULL), _current_pending_monitor_is_from_java(true), @@ -1082,6 +1102,7 @@ JavaThread::JavaThread() : _coroutine_stack_list(nullptr), _coroutine_list(nullptr), _current_coroutine(nullptr), + _wisp_preempt(false), _coroutine_temp(0), _handshake(this), @@ -1153,8 +1174,11 @@ void JavaThread::interrupt() { _ParkEvent->unpark(); } - bool JavaThread::is_interrupted(bool clear_interrupted) { + JavaThread* thread = JavaThread::current(); + if (UseWispMonitor && thread->is_Wisp_thread()) { + thread = ((WispThread*) thread)->thread(); + } debug_only(check_for_dangling_thread_pointer(this);) if (_threadObj.peek() == NULL) { @@ -1339,6 +1363,10 @@ void JavaThread::thread_main_inner() { this->set_native_thread_name(this->get_thread_name()); } HandleMark hm(this); + if (EnableCoroutine && !is_Compiler_thread()) { + // compiler thread never calls back into java + Coroutine::initialize_coroutine_support(this); + } this->entry_point()(this, this); } @@ -1488,6 +1516,27 @@ void JavaThread::exit(bool destroy_vm, ExitType exit_type) { assert(!this->has_pending_exception(), "release_monitors should have cleared"); } + if (EnableCoroutine && + // SurrogateLockerThread, JvmtiAgentThread, ServiceThread, CompilerThread + // are extended from JavaThread, but their entries are not thread_entry hence + // coroutineSupport was not initialized. We should not call `destroyCoroutineSupport` here. + !is_Compiler_thread() && + !is_hidden_from_external_view() && + // SurrogateLockerThread and ServiceThread are "is_hidden_from_external_view()" + !is_jvmti_agent_thread()) { + assert(!UseWispMonitor || destroy_vm || + java_lang_Thread::park_event(_threadObj), "park_event should been set"); + EXCEPTION_MARK; + JavaValue result(T_VOID); + JavaCalls::call_virtual(&result, + threadObj, vmClasses::Thread_klass(), + vmSymbols::destroyCoroutineSupport_method_name(), + vmSymbols::void_method_signature(), THREAD); + assert(_current_coroutine == _coroutine_list, "not thread coroutine"); + assert(_coroutine_list->next() == _coroutine_list, "ensure all coroutine has benn killed"); + CLEAR_PENDING_EXCEPTION; + } + // These things needs to be done while we are still a Java Thread. Make sure that thread // is in a consistent state, in case GC happens JFR_ONLY(Jfr::on_thread_exit(this);) @@ -2047,6 +2096,9 @@ void JavaThread::oops_do_no_frames(OopClosure* f, CodeBlobClosure* cf) { // Traverse instance variables at the end since the GC may be moving things // around using this function f->do_oop((oop*) &_vm_result); + if (EnableCoroutine) { + f->do_oop((oop*) &_vm_result_for_wisp); + } f->do_oop((oop*) &_exception_oop); f->do_oop((oop*) &_pending_async_exception); #if INCLUDE_JVMCI @@ -2704,6 +2756,14 @@ static void call_initPhase3(TRAPS) { JavaValue result(T_VOID); JavaCalls::call_static(&result, klass, vmSymbols::initPhase3_name(), vmSymbols::void_method_signature(), CHECK); + + if (EnableCoroutine) { + call_initializeWispClass(CHECK); + call_startWispDaemons(THREAD); + if (HAS_PENDING_EXCEPTION) { + vm_exit_during_initialization(Handle(THREAD, PENDING_EXCEPTION)); + } + } } void Threads::initialize_java_lang_classes(JavaThread* main_thread, TRAPS) { @@ -2747,6 +2807,10 @@ void Threads::initialize_java_lang_classes(JavaThread* main_thread, TRAPS) { // Phase 1 of the system initialization in the library, java.lang.System class initialization call_initPhase1(CHECK); + if (EnableCoroutine) { + initialize_class(vmSymbols::java_dyn_CoroutineSupport(), CHECK); + Coroutine::initialize_coroutine_support((JavaThread*) THREAD); + } // Get the Java runtime name, version, and vendor info after java.lang.System is initialized. // Some values are actually configure-time constants but some can be set via the jlink tool and // so must be read dynamically. We treat them all the same. @@ -3780,7 +3844,19 @@ JavaThread *Threads::owning_thread_from_monitor_owner(ThreadsList * t_list, DO_JAVA_THREADS(t_list, p) { // first, see if owner is the address of a Java thread - if (owner == (address)p) return p; + if (UseWispMonitor) { + if (p->coroutine_list()) { + Coroutine* c = p->coroutine_list(); + do { + if ((address) c->wisp_thread() == owner) { + return c->wisp_thread(); + } + c = c->next(); + } while (c != p->coroutine_list()); + } + } else if (owner == (address)p) { + return p; + } } // Cannot assert on lack of success here since this function may be @@ -3794,7 +3870,18 @@ JavaThread *Threads::owning_thread_from_monitor_owner(ThreadsList * t_list, // JavaThread* the_owner = NULL; DO_JAVA_THREADS(t_list, q) { - if (q->is_lock_owned(owner)) { + if (UseWispMonitor) { + if (q->coroutine_list()) { + Coroutine* c = q->coroutine_list(); + do { + if (c->wisp_thread()->is_lock_owned(owner)) { + the_owner = c->wisp_thread(); + break; + } + c = c->next(); + } while (c != q->coroutine_list()); + } + } else if (q->is_lock_owned(owner)) { the_owner = q; break; } @@ -3852,6 +3939,14 @@ void Threads::print_on(outputStream* st, bool print_stacks, p->trace_stack(); } else { p->print_stack_on(st); + if (EnableCoroutine) { + assert(p->coroutine_list() != NULL, "coroutine list"); + Coroutine* c = p->coroutine_list(); + do { + c->print_stack_on(st); + c = c->next(); + } while (c != p->coroutine_list()); + } } } st->cr(); diff --git a/src/hotspot/share/runtime/thread.hpp b/src/hotspot/share/runtime/thread.hpp index 4d7c5576730..543558a5ce2 100644 --- a/src/hotspot/share/runtime/thread.hpp +++ b/src/hotspot/share/runtime/thread.hpp @@ -93,6 +93,7 @@ class JavaThread; class Coroutine; class CoroutineStack; +class WispThread; // Class hierarchy // - Thread @@ -335,6 +336,7 @@ class Thread: public ThreadShadow { // Testers virtual bool is_VM_thread() const { return false; } virtual bool is_Java_thread() const { return false; } + virtual bool is_Wisp_thread() const { return false; } virtual bool is_Compiler_thread() const { return false; } virtual bool is_Code_cache_sweeper_thread() const { return false; } virtual bool is_service_thread() const { return false; } @@ -705,7 +707,6 @@ class JavaThread: public Thread { bool _on_thread_list; // Is set when this JavaThread is added to the Threads list OopHandle _threadObj; // The Java level thread object -#ifdef ASSERT private: int _java_call_counter; @@ -717,7 +718,6 @@ class JavaThread: public Thread { _java_call_counter--; } private: // restore original namespace restriction -#endif // ifdef ASSERT JavaFrameAnchor _anchor; // Encapsulation of current java frame and it state @@ -749,6 +749,7 @@ class JavaThread: public Thread { // Used to pass back results to the interpreter or generated code running Java code. oop _vm_result; // oop result is GC-preserved Metadata* _vm_result_2; // non-oop result + oop _vm_result_for_wisp; // this oop result is only for java call in _monitorexit // See ReduceInitialCardMarks: this holds the precise space interval of // the most recent slow path allocation for which compiled code has @@ -1042,6 +1043,7 @@ class JavaThread: public Thread { CoroutineStack* _coroutine_stack_list; Coroutine* _coroutine_list; Coroutine* _current_coroutine; + bool _wisp_preempt; intptr_t _coroutine_temp; @@ -1051,12 +1053,15 @@ class JavaThread: public Thread { CoroutineStack*& coroutine_stack_list() { return _coroutine_stack_list; } Coroutine*& coroutine_list() { return _coroutine_list; } Coroutine* current_coroutine() { return _current_coroutine; } + void set_current_coroutine(Coroutine *coro) { _current_coroutine = coro; } + void set_wisp_preempt(bool b) { _wisp_preempt = b; } static ByteSize coroutine_temp_offset() { return byte_offset_of(JavaThread, _coroutine_temp); } static ByteSize current_coroutine_offset() { return byte_offset_of(JavaThread, _current_coroutine); } - + static ByteSize wisp_preempt_offset() { return byte_offset_of(JavaThread, _wisp_preempt); } void initialize_coroutine_support(); + bool is_expected_thread_entry(ThreadFunction entry_point) { return _entry_point == entry_point; } private: friend class VMThread; @@ -1252,6 +1257,8 @@ class JavaThread: public Thread { Metadata* vm_result_2() const { return _vm_result_2; } void set_vm_result_2 (Metadata* x) { _vm_result_2 = x; } + oop vm_result_for_wisp() const { return _vm_result_for_wisp; } + void set_vm_result_for_wisp (oop x) { _vm_result_for_wisp = x; } MemRegion deferred_card_mark() const { return _deferred_card_mark; } void set_deferred_card_mark(MemRegion mr) { _deferred_card_mark = mr; } @@ -1317,6 +1324,7 @@ class JavaThread: public Thread { static ByteSize callee_target_offset() { return byte_offset_of(JavaThread, _callee_target); } static ByteSize vm_result_offset() { return byte_offset_of(JavaThread, _vm_result); } static ByteSize vm_result_2_offset() { return byte_offset_of(JavaThread, _vm_result_2); } + static ByteSize vm_result_for_wisp_offset() { return byte_offset_of(JavaThread, _vm_result_for_wisp ); } static ByteSize thread_state_offset() { return byte_offset_of(JavaThread, _thread_state); } static ByteSize polling_word_offset() { return byte_offset_of(JavaThread, _poll_data) + byte_offset_of(SafepointMechanism::ThreadData, _polling_word);} static ByteSize polling_page_offset() { return byte_offset_of(JavaThread, _poll_data) + byte_offset_of(SafepointMechanism::ThreadData, _polling_page);} @@ -1347,9 +1355,7 @@ class JavaThread: public Thread { } static ByteSize suspend_flags_offset() { return byte_offset_of(JavaThread, _suspend_flags); } -#ifdef ASSERT static ByteSize java_call_counter_offset() { return byte_offset_of(JavaThread, _java_call_counter); } -#endif static ByteSize do_not_unlock_if_synchronized_offset() { return byte_offset_of(JavaThread, _do_not_unlock_if_synchronized); } static ByteSize should_post_on_exceptions_flag_offset() { @@ -1499,6 +1505,9 @@ class JavaThread: public Thread { void thread_main_inner(); virtual void post_run(); + public: +// static ByteSize privileged_stack_top_offset() { return byte_offset_of(JavaThread, _privileged_stack_top); } + public: // Thread local information maintained by JVMTI. void set_jvmti_thread_state(JvmtiThreadState *value) { _jvmti_thread_state = value; } diff --git a/src/hotspot/share/runtime/thread.inline.hpp b/src/hotspot/share/runtime/thread.inline.hpp index 63101e77855..ff50e706201 100644 --- a/src/hotspot/share/runtime/thread.inline.hpp +++ b/src/hotspot/share/runtime/thread.inline.hpp @@ -142,7 +142,7 @@ inline JavaThreadState JavaThread::thread_state() const { } inline void JavaThread::set_thread_state(JavaThreadState s) { - assert(current_or_null() == NULL || current_or_null() == this, + assert(UseWispMonitor || current_or_null() == NULL || current_or_null() == this, "state change should only be called by the current thread"); #if defined(PPC64) || defined (AARCH64) // Use membars when accessing volatile _thread_state. See diff --git a/src/hotspot/share/runtime/vframe.cpp b/src/hotspot/share/runtime/vframe.cpp index 7669b19be7c..8b4a0482c22 100644 --- a/src/hotspot/share/runtime/vframe.cpp +++ b/src/hotspot/share/runtime/vframe.cpp @@ -248,7 +248,7 @@ void javaVFrame::print_lock_info_on(outputStream* st, int frame_count) { // the monitor is associated with an object, i.e., it is locked const char *lock_state = "locked"; // assume we have the monitor locked - if (!found_first_monitor && frame_count == 0) { + if (!found_first_monitor && frame_count == 0 || UseWispMonitor) { // If this is the first frame and we haven't found an owned // monitor before, then we need to see if we have completed // the lock or if we are blocked trying to acquire it. Only diff --git a/src/hotspot/share/runtime/vframe.hpp b/src/hotspot/share/runtime/vframe.hpp index f736bc71783..bf9473ff2d7 100644 --- a/src/hotspot/share/runtime/vframe.hpp +++ b/src/hotspot/share/runtime/vframe.hpp @@ -339,6 +339,10 @@ class vframeStream : public vframeStreamCommon { // Constructors vframeStream(JavaThread* thread, bool stop_at_java_call_stub = false, bool process_frames = true); + Thread *& thread_ref() { + return (Thread *&)_thread; + } + // top_frame may not be at safepoint, start with sender vframeStream(JavaThread* thread, frame top_frame, bool stop_at_java_call_stub = false); }; diff --git a/src/hotspot/share/services/threadService.hpp b/src/hotspot/share/services/threadService.hpp index be16a9684bf..8b942ce0e29 100644 --- a/src/hotspot/share/services/threadService.hpp +++ b/src/hotspot/share/services/threadService.hpp @@ -433,6 +433,9 @@ class JavaThreadStatusChanger : public StackObj { public: static void set_thread_status(JavaThread* java_thread, JavaThreadStatus state) { + if (UseWispMonitor && java_thread->is_Wisp_thread()) { + return; + } java_lang_Thread::set_thread_status(java_thread->threadObj(), state); } @@ -463,6 +466,8 @@ class JavaThreadStatusChanger : public StackObj { bool is_alive() { return _is_alive; } + + Thread *& thread_ref() { return (Thread *&)_java_thread; } }; // Change status to waiting on an object (timed or indefinite) diff --git a/src/hotspot/share/utilities/exceptions.cpp b/src/hotspot/share/utilities/exceptions.cpp index bd95b8306be..3f6ef1deff4 100644 --- a/src/hotspot/share/utilities/exceptions.cpp +++ b/src/hotspot/share/utilities/exceptions.cpp @@ -34,6 +34,7 @@ #include "memory/universe.hpp" #include "oops/oop.inline.hpp" #include "runtime/handles.inline.hpp" +#include "runtime/coroutine.hpp" #include "runtime/init.hpp" #include "runtime/java.hpp" #include "runtime/javaCalls.hpp" @@ -181,6 +182,9 @@ void Exceptions::_throw(JavaThread* thread, const char* file, int line, Handle h void Exceptions::_throw_msg(JavaThread* thread, const char* file, int line, Symbol* name, const char* message, Handle h_loader, Handle h_protection_domain) { + if (UseWispMonitor && thread->is_Wisp_thread()) { + thread = ((WispThread*) thread)->thread(); + } // Check for special boot-strapping/compiler-thread handling if (special_exception(thread, file, line, name, message)) return; // Create and throw exception @@ -287,6 +291,11 @@ Handle Exceptions::new_exception(JavaThread* thread, Symbol* name, signature, args, thread); + + { + guarantee(!EnableSteal || thread == Thread::current(), "fatal: stealed"); + } + } // Check if another exception was thrown in the process, if so rethrow that one @@ -319,6 +328,11 @@ Handle Exceptions::new_exception(JavaThread* thread, Symbol* name, vmSymbols::throwable_throwable_signature(), &args1, thread); + + { + guarantee(!EnableSteal || thread == Thread::current(), "fatal: stealed"); + } + } // Check if another exception was thrown in the process, if so rethrow that one diff --git a/src/java.base/linux/classes/sun/nio/ch/EPoll.java b/src/java.base/linux/classes/sun/nio/ch/EPoll.java index 6b3387d5728..2f17a2e2063 100644 --- a/src/java.base/linux/classes/sun/nio/ch/EPoll.java +++ b/src/java.base/linux/classes/sun/nio/ch/EPoll.java @@ -26,6 +26,8 @@ package sun.nio.ch; import java.io.IOException; + +import jdk.internal.access.SharedSecrets; import jdk.internal.misc.Unsafe; /** @@ -66,6 +68,8 @@ private EPoll() { } // flags static final int EPOLLONESHOT = (1 << 30); + static final int ENOENT; + /** * Allocates a poll array to handle up to {@code count} events. */ @@ -116,7 +120,51 @@ static int getEvents(long eventAddress) { static native int wait(int epfd, long pollAddress, int numfds, int timeout) throws IOException; + static native int errnoENOENT(); + static { IOUtil.load(); + ENOENT = errnoENOENT(); + SharedSecrets.setEpollAccess(new EpollAccess() { + @Override + public long allocatePollArray(int count) { + return EPoll.allocatePollArray(count); + } + + @Override + public void freePollArray(long address) { + EPoll.freePollArray(address); + } + + @Override + public long getEvent(long address, int i) { + return EPoll.getEvent(address, i); + } + + @Override + public int getDescriptor(long eventAddress) { + return EPoll.getDescriptor(eventAddress); + } + + @Override + public int getEvents(long eventAddress) { + return EPoll.getEvents(eventAddress); + } + + @Override + public int epollCreate() throws IOException { + return EPoll.create(); + } + + @Override + public int epollCtl(int epfd, int opcode, int fd, int events) { + return EPoll.ctl(epfd, opcode, fd, events); + } + + @Override + public int epollWait(int epfd, long pollAddress, int numfds) throws IOException { + return EPoll.wait(epfd, pollAddress, numfds, -1); + } + }); } } diff --git a/src/java.base/linux/classes/sun/nio/ch/EPollSelectorImpl.java b/src/java.base/linux/classes/sun/nio/ch/EPollSelectorImpl.java index 61e81956858..8ff0667787c 100644 --- a/src/java.base/linux/classes/sun/nio/ch/EPollSelectorImpl.java +++ b/src/java.base/linux/classes/sun/nio/ch/EPollSelectorImpl.java @@ -25,6 +25,10 @@ package sun.nio.ch; +import com.alibaba.wisp.engine.WispEngine; +import jdk.internal.access.SharedSecrets; +import jdk.internal.access.WispEngineAccess; + import java.io.IOException; import java.nio.channels.ClosedSelectorException; import java.nio.channels.SelectionKey; @@ -49,6 +53,8 @@ class EPollSelectorImpl extends SelectorImpl { + private static final WispEngineAccess WEA = SharedSecrets.getWispEngineAccess(); + // maximum number of events to poll in one call to epoll_wait private static final int NUM_EPOLLEVENTS = Math.min(IOUtil.fdLimit(), 1024); @@ -115,7 +121,9 @@ protected int doSelect(Consumer action, long timeout) do { long startTime = timedPoll ? System.nanoTime() : 0; - numEntries = EPoll.wait(epfd, pollArrayAddress, NUM_EPOLLEVENTS, to); + numEntries = WispEngine.transparentWispSwitch() ? + handleEPollWithWisp(to) : + EPoll.wait(epfd, pollArrayAddress, NUM_EPOLLEVENTS, to); if (numEntries == IOStatus.INTERRUPTED && timedPoll) { // timed poll interrupted so need to adjust timeout long adjust = System.nanoTime() - startTime; @@ -135,6 +143,20 @@ protected int doSelect(Consumer action, long timeout) return processEvents(numEntries, action); } + private int handleEPollWithWisp(long timeout) throws IOException { + if (timeout != 0 && WEA.usingWispEpoll(Thread.currentThread())) { + final int updated = EPoll.wait(epfd, pollArrayAddress, NUM_EPOLLEVENTS, 0); + if (updated > 0) { + return updated; + } + WEA.registerEpollEvent(epfd); + WEA.park(TimeUnit.MILLISECONDS.toNanos(timeout)); + return EPoll.wait(epfd, pollArrayAddress, NUM_EPOLLEVENTS, 0); + } else { + return EPoll.wait(epfd, pollArrayAddress, NUM_EPOLLEVENTS, (int) timeout); + } + } + /** * Process changes to the interest ops. */ diff --git a/src/java.base/linux/classes/sun/nio/ch/EPollSelectorProvider.java b/src/java.base/linux/classes/sun/nio/ch/EPollSelectorProvider.java index 9a3c5d2d036..0b39b1640ac 100644 --- a/src/java.base/linux/classes/sun/nio/ch/EPollSelectorProvider.java +++ b/src/java.base/linux/classes/sun/nio/ch/EPollSelectorProvider.java @@ -25,6 +25,10 @@ package sun.nio.ch; +import com.alibaba.wisp.engine.WispEngine; +import com.alibaba.wisp.util.io.SleepSelector; +import jdk.internal.access.SharedSecrets; + import java.io.IOException; import java.nio.channels.*; import java.nio.channels.spi.*; @@ -33,6 +37,15 @@ public class EPollSelectorProvider extends SelectorProviderImpl { public AbstractSelector openSelector() throws IOException { + AbstractSelector sel = new EPollSelectorImpl(this); + if (WispEngine.transparentWispSwitch() && + SharedSecrets.getWispEngineAccess().ifSpinSelector()) { + sel = new SleepSelector(sel); + } + return sel; + } + + public AbstractSelector openSelector0() throws IOException { return new EPollSelectorImpl(this); } diff --git a/src/java.base/linux/native/libnio/ch/EPoll.c b/src/java.base/linux/native/libnio/ch/EPoll.c index 1af7ef7a634..bbc8049cb81 100644 --- a/src/java.base/linux/native/libnio/ch/EPoll.c +++ b/src/java.base/linux/native/libnio/ch/EPoll.c @@ -94,3 +94,8 @@ Java_sun_nio_ch_EPoll_wait(JNIEnv *env, jclass clazz, jint epfd, } return res; } + +JNIEXPORT jint JNICALL +Java_sun_nio_ch_EPoll_errnoENOENT(JNIEnv *env, jclass this) { + return (jint)ENOENT; +} diff --git a/src/java.base/share/classes/com/alibaba/wisp/engine/ScheduledWispEngine.java b/src/java.base/share/classes/com/alibaba/wisp/engine/ScheduledWispEngine.java new file mode 100644 index 00000000000..6d79f978c0a --- /dev/null +++ b/src/java.base/share/classes/com/alibaba/wisp/engine/ScheduledWispEngine.java @@ -0,0 +1,621 @@ +package com.alibaba.wisp.engine; + +import sun.nio.ch.EPollSelectorProvider; + +import java.io.IOException; +import java.nio.channels.*; +import java.nio.channels.spi.SelectorProvider; +import java.util.*; +import java.util.concurrent.ConcurrentLinkedDeque; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; + +import jdk.internal.access.SharedSecrets; + +/** + * Per-thread self-scheduled implementation of WispEngine. + * These tasks are bound to Engine after they are created. + * We could create tasks in different WispEngine to make maximum use of multiprocessing. + *

+ * Here's how {@link java.net.Socket} cooperation with Java's NIO. + * +----------------------------------------------------------------------------------------------------+ + * | | + * | WispEngine.current() +---------------+ | + * | +->| nioSelector | ------------------------+ | + * | | +---------------+ <---------------------+ | | + * | | | | | + * | | | | | + * | | | | | + * | | | | | + * | +----------+ block on | +----------+ block on | v | + * | | wispTask | --------- socket | wispTask | ---------- socket | + * | +----------+ +----------+ | + * | this task use bio process one connection | + * | | + * | +----------+ block on +----------+ | + * | | wispTask | --------- socket | wispTask | | + * | +----------+ +----------+ | + * | | + * | +----------+ +----------+ | + * | | wispTask | socket | wispTask | | + * | +----------+ +----------+ | + * | | + * | Task may not block on socket or wispSelector (may be park(), sleep etc..). | + * | And Socket may not related on wispTask (in connection pool etc..). | + * | | + * +----------------------------------------------------------------------------------------------------+ + */ +final class ScheduledWispEngine extends WispEngine { + + @Override + protected void preloadClasses() throws Exception { + ScheduledWispEngine engine = ((ScheduledWispEngine) WispEngine.current()); + engine.selector(true); + WispTask.jdkPark(TimeUnit.MILLISECONDS.toNanos(1)); + + engine.selector.close(); + engine.selector = null; + } + + private Selector newSelector() { + boolean isInCritical0 = isInCritical; + isInCritical = true; + try { + SelectorProvider p = SelectorProvider.provider(); + if (p instanceof EPollSelectorProvider) + return ((EPollSelectorProvider) p).openSelector0(); + else + throw new IllegalStateException("unexpected provider"); + + } catch (IOException e) { + throw new IllegalStateException(e); + } finally { + isInCritical = isInCritical0; + } + } + + private Selector selector; + private AtomicBoolean wakened = new AtomicBoolean(true); + private Queue pendingTaskQueue = new ArrayDeque<>(); + + @Override + public void execute(Runnable target) { + if (JLA.currentThread0() == thread) { + runTaskInternal(target, "executor task", null, current.ctxClassLoader); + } else { + + + /* + I wanna create real task here, but coroutine's API + push users create coroutine at target thread. + So we create a fake coroutine, and let the engine + create real coroutine while got this pseudo-coroutine + in taskQueue. + @see {@link #processWakeTask(WispTask)} + */ + ClassLoader ctxClassLoader = current.ctxClassLoader; + WispTask pseudo = new WispTask(this, null, false, false); + long enqueueTime = getNanoTimeForProfile(); + pseudo.command = () -> { + if (runningTasks.size() >= WispConfiguration.EXTERNAL_SUBMIT_THRESHOLD) { + pendingTaskQueue.offer(pseudo); + return false; + } else { + WispEngine.current().countEnqueueTime(enqueueTime); + runTaskInternal(target, "executor task", null, ctxClassLoader); + return true; // means real switch happened + } + }; + wakeupTask(pseudo); + } + } + + @Override + protected void dispatchTask(Runnable target, String name) { + runTaskInternal(target, name, null, current.ctxClassLoader); + } + + @Override + WispTask getTaskFromCache() { + return taskCache.isEmpty() ? null : taskCache.remove(taskCache.size() - 1); + } + + @Override + void returnTaskToCache(WispTask task) { + taskCache.add(task); + } + + /** + * Put wakened task in this queue, and execute the task before this engine gets blocked on Selector + */ + private Deque wakeupQueue = new ConcurrentLinkedDeque<>(); + private TimeOut.TimerManager timerManager = new TimeOut.TimerManager(); + private Iterator eventIterator; + private List yieldTasks = new ArrayList<>(); + + @Override + protected boolean wakeupTask(WispTask task, boolean front) { + WispEngine engine = WispEngine.current(); + boolean isInCritical0 = engine.isInCritical; + engine.isInCritical = true; + try { + boolean needWakeup = task != null && !task.enqueued.get() && + task.enqueued.compareAndSet(false, true); + if (needWakeup) { + if (front) { + wakeupQueue.addFirst(task); + } else { + wakeupQueue.addLast(task); + } + } else { + statistics.alreadyWakeup++; + } + task.updateEnqueueTime(); + this.wakeup(); + return needWakeup; + } finally { + engine.isInCritical = isInCritical0; + } + } + + @Override + protected void yield() { + if (!runningTasks.isEmpty() || !wakeupQueue.isEmpty()) { + // for history reason, thread-coroutine is not + // treated as one runningTask, engine.runningTasks == 0 + // means only thread-coroutine, keep the raw thread behavior + if (selector == null || selector.keys().isEmpty()) { + wakeupTask(current, false); // append to wakeup queue's tail + } else { + // try best to run IO blocked tasks so that we put yielded + // tasks in separate queue which will be handled at end of doSelect + yieldTasks.add(current); + } + yieldOnBlocking(); // and do schedule + } else { + SharedSecrets.getJavaLangAccess().yield0(); + } + } + + @Override + protected void yieldToNext() { + int rescheduleCnt = 0; + + while (true) { + statistics.maxReschedule = Math.max(rescheduleCnt++, statistics.maxReschedule); + WispTask task; + + if (eventIterator != null && eventIterator.hasNext()) { + // 1. check completed IO + task = ((WispTask) eventIterator.next().attachment()); + eventIterator.remove(); + if (yieldTo(task)) { + statistics.doIO++; + break; + } + } else if ((runningTasks.size() < WispConfiguration.EXTERNAL_SUBMIT_THRESHOLD + && (task = pendingTaskQueue.poll()) != null) + || (task = wakeupQueue.poll()) != null) { + // 2. check wakened task + if (processWakeTask(task)) { + statistics.doWakeup++; + break; + } + } else { + // 3. check timer queue + TimeOut timeOut; + if ((timeOut = timerManager.queue.peek()) != null && + timeOut.deadlineNano <= System.nanoTime()) { + if (yieldTo(timerManager.queue.poll().task)) { + statistics.doTimer++; + counter.incrementTimeOutCount(); + break; + } + } else { + /* + * Another thread may call {@link #wakeup()} here, therefore + * if we don't check again, the last {@link #wakeupTask(WispTask)} will get lost. + */ + wakened.set(false); + + // double check unpark queue + if ((task = wakeupQueue.poll()) != null) { + statistics.doubleCheckWakeup++; + wakened.lazySet(true); + if (processWakeTask(task)) { + statistics.doWakeup++; + break; + } + } + /* + * {@link #wakeupTask(WispTask)} here is ok: + * The {@link #wakeupTask(WispTask)} call will do a {@link Selector#wakeup() , + * to ensure the pump returns immediately. + */ + // nothing to do ? call select, and retry + try { + long ms = -1; + if (0 != timerManager.queue.size()) { + long nanoDiff = timerManager.queue.peek().deadlineNano - System.nanoTime(); + ms = TimeUnit.NANOSECONDS.toMillis(nanoDiff + TimeUnit.MILLISECONDS.toNanos(1) / 2); + if (ms < 0) { + ms = 0; + } + if (WispConfiguration.PARK_ONE_MS_AT_LEAST && ms == 0) { + ms = 1; + } + } + if (!yieldTasks.isEmpty()) { + ms = 0; + } + doSelect(ms); + } catch (IOException e) { + // pass and retry + } + } + } + } + } + + private boolean processWakeTask(WispTask task) { + task.enqueued.lazySet(false); + if (task.command != null) { + return task.command.get(); + } else { + countEnqueueTime(task.getEnqueueTime()); + task.resetEnqueueTime(); + return yieldTo(task); + } + } + + @Override + protected void addTimer(long deadlineNano, boolean fromJvm) { + TimeOut timeOut = new TimeOut(current, deadlineNano, fromJvm); + current.timeOut = timeOut; + timerManager.addTimer(timeOut); + } + + @Override + protected void cancelTimer() { + if (current.timeOut != null) { + current.timeOut.canceled = true; + timerManager.cancelTimer(current.timeOut); + current.timeOut = null; + } + } + + @Override + protected void registerEvent(WispTask target, SelectableChannel ch, int events) + throws IOException { + + if (ch == null || !ch.isOpen()) + return; + + boolean isInCritical0 = isInCritical; + isInCritical = true; + try { + Selector sel = selector(false); + if (sel == null) { + if (events != 0) { // wispPoller is one shot, do not need clear + WispPoller.INSTANCE.registerEvent(target, ch, events); + } + return; + } + + int retry = 4; + while (retry-- > 0) { + try { + // detach old interest ch + if (target.ch != null && target.ch != ch) { + SelectionKey key = target.ch.keyFor(sel); + if (key != null && key.isValid() && key.interestOps() != 0 && + key.attachment() == target) { + key.interestOps(0); // may produce CancelledKeyException + } + } + + SelectionKey key = ch.keyFor(sel); + if (key != null) { + // nio put canceled key to a list, + // and process it next loop. so retry + if (!key.isValid()) { + statistics.retryCanceledKey++; + doSelect(0); + continue; + } + if (events == 0) { + if (key.attachment() == target) { + key.interestOps(0); // may produce CancelledKeyException + } + } else { + if (key.interestOps() != events) { + key.interestOps(events); // may produce CancelledKeyException + } + if (key.attachment() != target) { + key.attach(target); + } + } + } else { + ch.register(sel, events, target); + } + target.ch = ch; + return; + } catch (CancelledKeyException e) { + statistics.retryCanceledKey++; + doSelect(0); + } catch (ClosedChannelException e) { + // pass and retry + } + } + throw new IOException(); + } finally { + isInCritical = isInCritical0; + } + } + + + // record if the NIO consume 100% CPU BUG happened + private static final int REBUILD_THRESHOLD = 128; + private int rebuildStatus = 0; + + // record idle rate + private long selectTimestamp; + private long cycleDuration; + private int idlePerK = 1000; + + /** + * Wake up the engine that is blocked on {@link Selector#select()}. + *

+ * {@link Selector#wakeup()} is an expensive operation(fd write), + * so use a status variable to reduce the cost. + */ + private void wakeup() { + statistics.wakeupCount++; + WispEngine self = WispEngine.current(); // self wakeup this + if (this != self) { + if (!wakened.get() && wakened.compareAndSet(false, true)) { + statistics.realWakeupCount++; + boolean isInCritical0 = self.isInCritical; + self.isInCritical = true; + if (selector == null) { + // selector may became non-null + // and use select() in the target engine + // see WispEngine#selector() + SharedSecrets.getUnsafeAccess().unpark0(thread); + } else { + selector.wakeup(); + } + self.isInCritical = isInCritical0; + } + } else { + statistics.innerEngineWakeup++; + } + } + + /** + * 1. count running status (select is the only way to block current thread) + * 2. handle NIO bug + * 3. dispatch event to proxied Selector + * + * @param ms == 0 -> selectNow() + * ms < 0 -> select() + * ms > 0 -> select(timeout) + * @return selectedCount + * @throws IOException + */ + int doSelect(long ms) throws IOException { + schedTick++; + Selector sel = selector; + statistics.eventLoops++; + counter.incrementEventLoopCount(); + int selectCnt = -1; + + boolean isInCritical0 = isInCritical; + isInCritical = true; + + try { + if (wakened.get() || ms == 0) { + statistics.nonBlockSelectCount++; + if (sel != null) { + selectCnt = sel.selectNow(); + } + } else { + + long startTs = System.nanoTime(); + + long busyTime = selectTimestamp == 0 ? 0 : startTs - selectTimestamp; + counter.incrementRunningTimeTotal(busyTime); + + boolean interrupted = current.isInterrupted(); + if (interrupted) { // avoid unexpected wakeup, see AbstractSelector.begin() + current.lazySetInterrupted(false); + } + // if a selector.wakeup() called before here, + // selector.select(...)' will wake up immediately. + if (sel != null) { + selectCnt = ms > 0 ? sel.select(ms) : sel.select(); + } else { + SharedSecrets.getUnsafeAccess().park0(false, ms < 0 ? 0 : TimeUnit.MILLISECONDS.toNanos(ms)); + } + if (interrupted) { + current.lazySetInterrupted(true); + } + + selectTimestamp = System.nanoTime(); + long idleTime = selectTimestamp - startTs; + counter.incrementWaitTime(idleTime); + long currentCycle = idleTime + busyTime; + int currentIdle = currentCycle == 0 ? 500 : + (int) (1000 * idleTime / (currentCycle)); + + int currentIdleRate = (int) ((idlePerK * cycleDuration + currentIdle * currentCycle) / + (currentCycle + cycleDuration)); + // weighted mean by cycle duration + idlePerK = (int) exponentialSmoothed(idlePerK, currentIdleRate); + cycleDuration = exponentialSmoothed(cycleDuration, currentCycle); + + + if (selectCnt == 0 && idleTime <= 100000 /* 0.1ms */ && !wakened.get()) { + // In ajdk 8.4.8, While unparking a thread(which actually waking up the underlying selector + // used by WispEngine), an IOException might occur if the unparking is issued on a 'closed' + // selector(which has been made as 'closed' after rebuilding) + // To solve it, before calling rebuildSelector, set wakened as true. Then selector.wakeup + // won't be called (see wakeup) + if (rebuildStatus++ > REBUILD_THRESHOLD && wakened.compareAndSet(false, true)) { + rebuildSelector(); + rebuildStatus = 0; + } + } else { + rebuildStatus = 0; + } + // reduce the cost of following {@link #wakeup} + wakened.lazySet(true); + } + + if (sel == null) { + return 0; + } + + if (selectCnt != 0) { + eventIterator = selector.selectedKeys().iterator(); + statistics.ioEventCount += selectCnt; + } else { + if (ms == 0) { + statistics.selectNothingActive++; + } else { + statistics.selectNothingPassive++; + } + } + + for (WispTask task : yieldTasks) { + wakeupTask(task, false); + } + yieldTasks.clear(); + + return selectCnt; + + } finally { + isInCritical = isInCritical0; + } + } + + Selector selector(boolean ensureCreate) { + if (selector == null && (!WispConfiguration.GLOBAL_POLLER || ensureCreate)) { + selector = newSelector(); + /* + * thread A sees selector == null in WispEngine.wakeup() + * thread B calls WispEngine.selector(true) and then selector.select() on newly created selector. + * A unparks for thread B via unsafe, but unfortunately it will fail as thread B currently is + * blocking on selector.select() (instead of parker) + * So we do selector.wakeup() to selector.select() to prevent this problem. + */ + selector.wakeup(); + } + return selector; + } + + @Override + protected boolean isRunning() { + return wakened.get(); + } + + @Override + protected int getTaskQueueLength() { + if (selector != null) { + return wakeupQueue.size() + pendingTaskQueue.size() + selector.selectedKeys().size(); + } else { + return wakeupQueue.size() + pendingTaskQueue.size(); + } + } + + + /** + * handle NIO consuming 100% CPU BUG + * see https://issues.apache.org/jira/browse/DIRMINA-678 + */ + private void rebuildSelector() throws IOException { + assert selector != null; + final Selector oldSelector = selector; + final Selector newSelector; + + try { + newSelector = newSelector(); + } catch (Exception e) { + throw new IOException(e); + } + + // Register all channels to the new Selector. + while (true) { + try { + for (SelectionKey key : oldSelector.keys()) { + Object a = key.attachment(); + try { + if (!key.isValid() || key.channel().keyFor(newSelector) != null) { + continue; + } + + int interestOps = key.interestOps(); + key.cancel(); + key.channel().register(newSelector, interestOps, a); + } catch (Exception e) { + throw new IOException(e); + } + } + } catch (ConcurrentModificationException e) { + // Probably due to concurrent modification of the key set. + continue; + } + + break; + } + + selector = newSelector; + statistics.selectorRebuild++; + + try { + oldSelector.close(); + } catch (Throwable t) { + // pass + } + } + + @Override + protected void closeEngineSelector() { + boolean isInCritical0 = isInCritical; + if (selector != null) { + try { + isInCritical = true; + selector.close(); + } catch (IOException e) { + // ignore + } finally { + isInCritical = isInCritical0; + } + } + } + + @Override + protected void startShutdown() { + threadTask.enqueued.lazySet(false); + // if threadTask has been enqueued, "switch to threadTask" + // may happen before `hasBeenShutdown` has been seen. + // ignore WispTask.enqueued flag, to ensure threadTask + // could be scheduled after `hasBeenShutdown` has been seen + + // let threadTask launch killing process + wakeupTask(threadTask, true); + } + + @Override + protected void iterateTasksForShutdown() { + while (!runningTasks.isEmpty()) { + for (WispTask t : runningTasks) { + wakeupTask(t); + } + // ensure we could come back again and + // check if all tasks has been exited + wakeupTask(current); + yieldToNext(); + } + } +} diff --git a/src/java.base/share/classes/com/alibaba/wisp/engine/StealAwareRunnable.java b/src/java.base/share/classes/com/alibaba/wisp/engine/StealAwareRunnable.java new file mode 100644 index 00000000000..cf0c236464b --- /dev/null +++ b/src/java.base/share/classes/com/alibaba/wisp/engine/StealAwareRunnable.java @@ -0,0 +1,21 @@ +package com.alibaba.wisp.engine; + + +/** + * An runnable that is aware of work stealing. + */ +public interface StealAwareRunnable extends Runnable { + + /** + * @return if that runnable could be stolen + */ + default boolean isStealEnable() { + return true; + } + + /** + * Set this runnable's {@link #isStealEnable} to given value + */ + default void setStealEnable(boolean b) { + } +} diff --git a/src/java.base/share/classes/com/alibaba/wisp/engine/TimeOut.java b/src/java.base/share/classes/com/alibaba/wisp/engine/TimeOut.java new file mode 100644 index 00000000000..dfb76a19d1c --- /dev/null +++ b/src/java.base/share/classes/com/alibaba/wisp/engine/TimeOut.java @@ -0,0 +1,221 @@ +package com.alibaba.wisp.engine; + +import java.util.Arrays; + +/** + * Represents {@link WispTask} related time out + */ +public class TimeOut { + final WispTask task; + long deadlineNano; + boolean canceled = false; + /** + * its position in the heapArray, -1 indicates that TimeOut has been deleted + */ + private final boolean fromJvm; + private int queueIdx; + private TimerManager manager; + + /** + * @param task related {@link WispTask} + * @param deadlineNano wake up related {@link WispTask} at {@code deadline} if not canceled + */ + public TimeOut(WispTask task, long deadlineNano, boolean fromJvm) { + this.task = task; + this.deadlineNano = deadlineNano; + this.fromJvm = fromJvm; + } + + /** + * @return {@code true} if and only if associated timer is expired. + */ + public boolean expired() { + return !canceled && System.nanoTime() >= deadlineNano; + } + + /** + * unpark the blocked task + */ + void doUnpark() { + if (fromJvm) { + task.unpark(); + } else { + task.jdkUnpark(); + } + } + + /** + * We use minimum heap algorithm to get the minimum deadline of all timers, + * also keep every TimeOut's queueIdx(its position in heap) so that we can easily + * remove it. + */ + static class TimerManager { + Queue queue = new Queue(); + + void addTimer(TimeOut timeOut) { + timeOut.deadlineNano = overflowFree(timeOut.deadlineNano, queue.peek()); + timeOut.manager = this; + queue.offer(timeOut); + } + + void cancelTimer(TimeOut timeOut) { + if (timeOut.manager == this && timeOut.queueIdx != -1) { + queue.remove(timeOut); + } + } + + /** + * Dispatch timeout events and return the timeout interval for next + * first timeout task + * + * @return -1: infinitely waiting, > 0 wait nanos + */ + long processTimeoutEventsAndGetWaitNanos() { + long nanos = -1; + TimeOut timeOut; + if (queue.size != 0) { + long now = System.nanoTime(); + while ((timeOut = queue.peek()) != null) { + if (timeOut.canceled) { + queue.poll(); + } else if (timeOut.deadlineNano <= now) { + queue.poll(); + timeOut.doUnpark(); + } else { + nanos = timeOut.deadlineNano - now; + break; + } + } + } + return nanos; + } + + class Queue { + private static final int INITIAL_CAPACITY = 16; + private TimeOut[] queue = new TimeOut[INITIAL_CAPACITY]; + private int size = 0; + + /** + * Inserts TimeOut x at position k, maintaining heap invariant by + * promoting x up the tree until it is greater than or equal to + * its parent, or is the root. + * + * @param k the position to fill + * @param timeOut the TimeOut to insert + */ + private void siftUp(int k, TimeOut timeOut) { + while (k > 0) { + int parent = (k - 1) >>> 1; + TimeOut e = queue[parent]; + if (timeOut.deadlineNano >= e.deadlineNano) { + break; + } + queue[k] = e; + e.queueIdx = k; + k = parent; + } + queue[k] = timeOut; + timeOut.queueIdx = k; + } + + /** + * Inserts item x at position k, maintaining heap invariant by + * demoting x down the tree repeatedly until it is less than or + * equal to its children or is a leaf. + * + * @param k the position to fill + * @param timeOut the item to insert + */ + private void siftDown(int k, TimeOut timeOut) { + int half = size >>> 1; + while (k < half) { + int child = (k << 1) + 1; + TimeOut c = queue[child]; + int right = child + 1; + if (right < size && c.deadlineNano > queue[right].deadlineNano) { + c = queue[child = right]; + } + if (timeOut.deadlineNano <= c.deadlineNano) { + break; + } + queue[k] = c; + c.queueIdx = k; + k = child; + } + queue[k] = timeOut; + timeOut.queueIdx = k; + } + + + public boolean remove(TimeOut timeOut) { + int i = timeOut.queueIdx; + if (i == -1) { + //this timeOut has been deleted. + return false; + } + + queue[i].queueIdx = -1; + int s = --size; + TimeOut replacement = queue[s]; + queue[s] = null; + if (s != i) { + siftDown(i, replacement); + if (queue[i] == replacement) { + siftUp(i, replacement); + } + } + return true; + } + + public int size() { + return size; + } + + public boolean offer(TimeOut timeOut) { + int i = size++; + if (i >= queue.length) { + queue = Arrays.copyOf(queue, queue.length * 2); + } + if (i == 0) { + queue[0] = timeOut; + timeOut.queueIdx = 0; + } else { + siftUp(i, timeOut); + } + return true; + } + + public TimeOut poll() { + if (size == 0) { + return null; + } + TimeOut f = queue[0]; + int s = --size; + TimeOut x = queue[s]; + queue[s] = null; + if (s != 0) { + siftDown(0, x); + } + f.queueIdx = -1; + return f; + } + + public TimeOut peek() { + return queue[0]; + } + } + } + + private static long overflowFree(long deadlineNano, TimeOut head) { + if (deadlineNano < Long.MIN_VALUE / 2) { // deadlineNano regarded as negative overflow + deadlineNano = Long.MAX_VALUE; + } + if (head != null && head.deadlineNano < 0 && deadlineNano > 0 && deadlineNano - head.deadlineNano < 0) { + deadlineNano = Long.MAX_VALUE + head.deadlineNano; + // then deadlineNano - head.deadlineNano = Long.MAX_VALUE > 0 + // i.e. deadlineNano > head.deadlineNano + } + return deadlineNano; + } +} + diff --git a/src/java.base/share/classes/com/alibaba/wisp/engine/Wisp2Engine.java b/src/java.base/share/classes/com/alibaba/wisp/engine/Wisp2Engine.java new file mode 100644 index 00000000000..7ebf6cf19f4 --- /dev/null +++ b/src/java.base/share/classes/com/alibaba/wisp/engine/Wisp2Engine.java @@ -0,0 +1,354 @@ +package com.alibaba.wisp.engine; + +import java.io.IOException; +import java.nio.channels.SelectableChannel; +import java.nio.channels.SelectionKey; +import java.nio.channels.Selector; +import java.nio.channels.spi.AbstractSelector; +import java.util.Queue; +import java.util.concurrent.*; + +/** + * In wisp1 WispTask could run in ANY user created thread. + * Wisp2 thread model has been changed to java.util.concurrent.Executor + *

+ * The new thread model bring some restrictions to us, brings us listed benefit: + *

    + *
  1. Work load could be steal by other threads in Executor
    2. + *
  2. Eliminate park/unpark API hook for non coroutine thread
    3. + *
+ */ +final class Wisp2Engine extends WispEngine { + + private static final ScheduledExecutorService timer = !WispConfiguration.WISP_HIGH_PRECISION_TIMER ? null : + Executors.newScheduledThreadPool(1, new ThreadFactory() { + @Override + public Thread newThread(Runnable r) { + Thread thread = new Thread(WispEngine.daemonThreadGroup, r); + thread.setDaemon(true); + thread.setName("Wisp2-Timer"); + return thread; + } + }); + + private static Queue globalTaskCache = new ConcurrentLinkedQueue<>(); + + @Override + protected void preloadClasses() throws Exception { + if (WispConfiguration.WISP_HIGH_PRECISION_TIMER) { + timer.submit(new Runnable() { + @Override + public void run() { + } + }); + } + addTimer(System.nanoTime() + Integer.MAX_VALUE, false); + cancelTimer(); + } + + Wisp2Group group; + Wisp2Scheduler.Carrier carrier; + + Wisp2Engine(Wisp2Group group) { + this.group = group; + } + + @Override + protected void postInit() { + runningTasks = new ConcurrentSkipListSet<>(); // support concurrent modify + } + + private TimeOut pendingTimer; + + @Override + protected void addTimer(long deadlineNano, boolean fromJvm) { + WispTask task = current; + TimeOut timeOut = new TimeOut(task, deadlineNano, fromJvm); + task.timeOut = timeOut; + + if (WispConfiguration.WISP_HIGH_PRECISION_TIMER) { + if (task.isThreadTask()) { + scheduleInTimer(timeOut); + } else { + /* + * timer.schedule may enter park() again + * we delegate this operation to thread coroutine + * (which always use native park) + */ + pendingTimer = timeOut; + } + } else { + group.scheduler.addTimer(timeOut, thread); + } + } + + private void processPendingTimer() { + if (pendingTimer != null) { + scheduleInTimer(pendingTimer); + pendingTimer = null; + } + } + + private void scheduleInTimer(TimeOut timeOut) { + boolean isInCritical0 = isInCritical; + final long timeout = timeOut.deadlineNano - System.nanoTime(); + isInCritical = true; + if (timeout > 0) { + timer.schedule(new Runnable() { + @Override + public void run() { + if (!timeOut.canceled) { + timeOut.doUnpark(); + } + } + }, timeout, TimeUnit.NANOSECONDS); + } else if (!timeOut.canceled) { + timeOut.task.jdkUnpark(); + } + isInCritical = isInCritical0; + } + + @Override + protected void cancelTimer() { + if (current.timeOut != null) { + current.timeOut.canceled = true; + group.scheduler.cancelTimer(current.timeOut, thread); + current.timeOut = null; + } + pendingTimer = null; + } + + /** + * Steal task from it's carrier engine to current + * + * @param failOnContention steal fail if there's too much lock contention + * @return if success + */ + private static boolean steal(WispTask task, WispEngine current, boolean failOnContention) { + assert current == WispEngine.current(); + if (task.engine != current) { + while (task.stealLock != 0) {/* wait until steal enabled */} + assert task.status != WispTask.Status.RUNNABLE; + assert task.parent == null; + if (!task.ctx.steal(failOnContention)) { + task.stealFailureCount++; + return false; + } + task.stealCount++; + task.engine = current; + } + return true; + } + + private static boolean resumeTask(WispTask task, boolean failOnContention) { + WispEngine current = WispEngine.current(); + /* + * Please be extremely cautious: + * task.engine can not be changed here by other thread + * is base on our implementation of using park instead of + * direct yieldOnBlocking, so only one thread could receive + * this closure. + */ + if (task.engine != current) { + WispEngine source = task.engine; + if (!steal(task, current, failOnContention)) { + return false; + } + source.runningTasks.remove(task); + current.runningTasks.add(task); + } + current.countEnqueueTime(task.getEnqueueTime()); + task.resetEnqueueTime(); + current.yieldTo(task); + current.runWispTaskEpilog(); + return true; + } + + @Override + StealAwareRunnable createResumeEntry(WispTask task) { + assert !task.isThreadTask(); + return new StealAwareRunnable() { + boolean stealEnable = true; + @Override + public void run() { + if (!resumeTask(task, true)) { + stealEnable = false; + wakeupTask(task); + } + } + + @Override + public void setStealEnable(boolean b) { + stealEnable = b; + } + + @Override + public boolean isStealEnable() { + return stealEnable; + } + }; + } + + /** + * @param task target task + * @param urgent not used + */ + @Override + protected boolean wakeupTask(WispTask task, boolean urgent) { + assert !task.isThreadTask(); + assert task.resumeEntry != null; + task.updateEnqueueTime(); + group.scheduler.executeWithCarrierThread(task.resumeEntry, task.engine.thread); + return true; + } + + @Override + protected void yieldToNext() { + assert current.resumeEntry != null; + current.resumeEntry.setStealEnable(true); + yieldTo(threadTask); // letting the scheduler choose runnable task + } + + @Override + protected void yield() { + if (WispEngine.runningAsCoroutine(current.getThreadWrapper())) { + wakeupTask(current); + // can be steal, and the stealing thread may + // blocking on `stealingLock` + // Not a big problem, because we'll release it + // immediately + yieldOnBlocking(); + } else { + JLA.yield0(); + } + } + + @Override + public void execute(Runnable target) { + dispatchTask(target, "executor task"); + } + + @Override + protected void dispatchTask(Runnable target, String name) { + // wisp2 DO NOT ALLOW running task in non-scheduler thread + ClassLoader ctxClassLoader = current.ctxClassLoader; + long enqueueTime = getNanoTimeForProfile(); + + group.scheduler.execute(new StealAwareRunnable() { + @Override + public void run() { + WispEngine current = WispEngine.current(); + current.countEnqueueTime(enqueueTime); + current.runTaskInternal(target, name, null, ctxClassLoader); + } + }); + } + + @Override + WispTask getTaskFromCache() { + if (!taskCache.isEmpty()) { + return taskCache.remove(taskCache.size() - 1); + } + WispTask task = globalTaskCache.poll(); + if (task == null) { + return null; + } + if (task.engine != this) { + if (!steal(task, this, true)) { + globalTaskCache.add(task); + return null; + } + } + return task; + } + + @Override + void returnTaskToCache(WispTask task) { + if (taskCache.size() > WispConfiguration.WISP_ENGINE_TASK_CACHE_SIZE) { + globalTaskCache.add(task); + } else { + taskCache.add(task); + } + } + + @Override + protected void registerEvent(WispTask target, SelectableChannel ch, int events) throws IOException { + if (ch != null && ch.isOpen() && events != 0) { + WispPoller.INSTANCE.registerEvent(target, ch, events); + } + } + + @Override + protected void registerEpollEvent(int epFd) throws IOException { + WispPoller.INSTANCE.registerEvent(current, epFd, SelectionKey.OP_READ); + } + + @Override + protected boolean isRunning() { + return current != threadTask; + } + + @Override + protected int getTaskQueueLength() { + return carrier == null ? 0 : (carrier.queueLength > 0 ? carrier.queueLength : 0); + } + + @Override + protected void runWispTaskEpilog() { + processPendingTimer(); + } + + @Override + protected void startShutdown() { + group.scheduler.executeWithCarrierThread(new StealAwareRunnable() { + @Override + public void run() { + doShutdown(); + } + + @Override + public boolean isStealEnable() { + return false; + } + }, thread); + } + + @Override + protected void iterateTasksForShutdown() { + while (!runningTasks.isEmpty()) { + yieldTo(runningTasks.iterator().next()); + } + } + + /** + * Current retake strategy is: + * 1. allocate a new WispEngine as the copy of the blocked engine + * 2. handOff the blocked engine + * 2. Let other engine steal all tasks + */ + @Override + protected void handOff() { + group.scheduler.handOffCarrierThread(thread); + // this carrier can not steal task from now on, + // runningTasks will never grow + for (WispTask task : runningTasks) { + if (current != task) { + group.scheduler.execute(new StealAwareRunnable() { + @Override + public void run() { + // avoiding steal failure by lock contention + if (!resumeTask(task, false)) { + // The task will be remained in the handoffed thread. + // We already make our best efforts. + // Forcedly stealing a steal-disabled task which contains + // an unexpected native frame will result in jvm crash + + // retry is meaningless, because the carrier thread is occupied + // by the running task, and other tasks has no chance to move on + } + } + }); + } + } + } +} diff --git a/src/java.base/share/classes/com/alibaba/wisp/engine/Wisp2Group.java b/src/java.base/share/classes/com/alibaba/wisp/engine/Wisp2Group.java new file mode 100644 index 00000000000..63be7cd8ab9 --- /dev/null +++ b/src/java.base/share/classes/com/alibaba/wisp/engine/Wisp2Group.java @@ -0,0 +1,128 @@ +package com.alibaba.wisp.engine; + +import java.util.Comparator; +import java.util.List; +import java.util.Set; +import java.util.concurrent.AbstractExecutorService; +import java.util.concurrent.ConcurrentSkipListSet; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; + +/** + * WispV2Group represents a series of {@link Wisp2Engine}, which can steal + * tasks from each other to achieve work-stealing. + *

+ * {@code WispV2Group#WISP2_ROOT_GROUP} is created by system. + * {@link WispEngine#dispatch(Runnable)} in non-carrier thread and WISP2_ROOT_GROUP's + * carrier thread will dispatch task in this group. + *

+ * User code could also create {@link Wisp2Group} by calling + * {@link Wisp2Group#createGroup(int, ThreadFactory)}, + * Calling {@link Wisp2Group#execute(Runnable)} will dispatch + * WispTask inner created group. + * {@link WispEngine#dispatch(Runnable)} in a user created group will also + * dispatch task in current group. + */ +public class Wisp2Group extends AbstractExecutorService { + + private static final String WISP2_ROOT_GROUP_NAME = "Root"; + + static final Wisp2Group WISP2_ROOT_GROUP = new Wisp2Group(WISP2_ROOT_GROUP_NAME); + + final Wisp2Scheduler scheduler; + final Set carrierEngines; + + /** + * Create a new WispV2Group for executing tasks. + * + * @param size carrier thread counter + * @param tf ThreadFactory used to create carrier thread + */ + public static Wisp2Group createGroup(int size, ThreadFactory tf) { + return new Wisp2Group(size, tf); + } + + private static Set createEngineSet() { + return new ConcurrentSkipListSet<>(new Comparator() { + @Override + public int compare(Wisp2Engine o1, Wisp2Engine o2) { + return o1.threadTask.compareTo(o2.threadTask); + } + }); + } + + /** + * Create Root Carrier. + */ + private Wisp2Group(String name) { + carrierEngines = createEngineSet(); + scheduler = new Wisp2Scheduler( + WispConfiguration.WORKER_COUNT, + WispConfiguration.WISP_SCHEDULE_STEAL_RETRY, + WispConfiguration.WISP_SCHEDULE_PUSH_RETRY, + WispConfiguration.WISP_SCHEDULE_HELP_STEAL_RETRY, + new ThreadFactory() { + AtomicInteger seq = new AtomicInteger(); + + @Override + public Thread newThread(Runnable r) { + Thread t = new Thread(r, "Wisp2-" + name + "-Carrier-" + seq.getAndIncrement()); + t.setDaemon(WispConfiguration.WISP_DAEMON_WORKER); + return t; + } + }, + this, false); + } + + private Wisp2Group(int size, ThreadFactory tf) { + carrierEngines = createEngineSet(); + scheduler = new Wisp2Scheduler(size, tf, this); + } + + @Override + public void shutdown() { + for (Wisp2Engine worker : carrierEngines) { + worker.shutdown(); + } + } + + @Override + public List shutdownNow() { + throw new UnsupportedOperationException(); + } + + @Override + public boolean isShutdown() { + return carrierEngines.iterator().next().isShutdown(); + } + + @Override + public boolean isTerminated() { + return carrierEngines.stream().allMatch(WispEngine::isTerminated); + } + + @Override + public boolean awaitTermination(long timeout, TimeUnit unit) throws InterruptedException { + long deadline = System.nanoTime() + unit.toNanos(timeout); + for (Wisp2Engine worker : carrierEngines) { + long t = deadline - System.nanoTime(); + if (t <= 0 || !worker.awaitTermination(t, TimeUnit.NANOSECONDS)) { + return false; + } + } + return true; + } + + @Override + public void execute(Runnable command) { + scheduler.execute(new StealAwareRunnable() { + @Override + public void run() { + WispEngine engine = WispEngine.current(); + engine.runTaskInternal(command, "group dispatch task", + null, engine.current.ctxClassLoader); + } + }); + } +} diff --git a/src/java.base/share/classes/com/alibaba/wisp/engine/Wisp2Scheduler.java b/src/java.base/share/classes/com/alibaba/wisp/engine/Wisp2Scheduler.java new file mode 100644 index 00000000000..a017053af77 --- /dev/null +++ b/src/java.base/share/classes/com/alibaba/wisp/engine/Wisp2Scheduler.java @@ -0,0 +1,350 @@ +package com.alibaba.wisp.engine; + +import jdk.internal.access.JavaLangAccess; +import jdk.internal.access.SharedSecrets; +import jdk.internal.access.UnsafeAccess; + +import java.util.Arrays; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.atomic.AtomicIntegerFieldUpdater; + +/** + * Wisp2 work-stealing implementation. + *

+ * Every carrier thread has a {@link ConcurrentLinkedQueue} to + * receive tasks. + * Once local queue is empty, carrier will scan + * others' queue, execute stolen task, then check local queue. + * Again and again, until all the queue is empty. + */ +class Wisp2Scheduler { + + private final static int IDX_MASK = 0xffff; // ensure positive + private final static int STEAL_HIGH_WATER_LEVEL = 8; + + // instance const + private final int PARALLEL; + private final int STEAL_RETRY; + private final int PUSH_RETRY; + private final int HELP_STEAL_RETRY; + + // carriers could be changed by handOff(), + // add volatile to avoiding carriers's elements + // to be allocated in register. + // we could not add volatile volatile modifier + // to array elements, so make the array volatile. + private volatile Carrier[] carriers; + private final ThreadFactory threadFactory; + private final Wisp2Group group; + + Wisp2Scheduler(int parallelism, ThreadFactory threadFactory, Wisp2Group group) { + this(parallelism, parallelism, parallelism, Math.max(1, parallelism / 2), threadFactory, group, true); + } + + Wisp2Scheduler(int parallelism, int stealRetry, int pushRetry, int helpStealRetry, + ThreadFactory threadFactory, Wisp2Group group, boolean startThreads) { + PARALLEL = parallelism; + STEAL_RETRY = stealRetry; + PUSH_RETRY = pushRetry; + HELP_STEAL_RETRY = helpStealRetry; + this.group = group; + this.threadFactory = threadFactory; + carriers = new Carrier[PARALLEL]; + for (int i = 0; i < parallelism; i++) { + carriers[i] = new Carrier(); + } + if (startThreads) { + startCarrierThreads(); + } + } + + void startCarrierThreads() { + for (Carrier carrier : carriers) { + carrier.thread.start(); + } + } + + class Carrier implements Runnable { + ConcurrentLinkedQueue taskQueue; + private final TimeOut.TimerManager timerManager; + private final Thread thread; + volatile boolean detached = false; + + private final static int QL_IDLE_PROCESSING_TIMER = -1; + private final static int QL_IDLE = -2; + volatile int queueLength; + + Carrier() { + thread = threadFactory.newThread(this); + WispEngine.carrierThreads.add(thread); + taskQueue = new ConcurrentLinkedQueue<>(); + timerManager = new TimeOut.TimerManager(); + queueLength = 0; + } + + void runCarrier(WispEngine engine) { + int r = (int) System.nanoTime(); + Runnable task; + while (true) { + while ((task = pollTask(false)) != null) { + doExec(task); + } + + if (detached) { + if (engine.runningTasks.isEmpty()) { + return; + } + } else if ((task = trySteal(STEAL_RETRY, r = nextRandom(r))) != null) { + doExec(task); + continue; // process local queue + } + + if (queueLength == 0 && LENGTH_UPDATER.compareAndSet(this, 0, QL_IDLE_PROCESSING_TIMER)) { + long nanos = timerManager.processTimeoutEventsAndGetWaitNanos(); + assert nanos != 0; + boolean doPark = false; + if (queueLength == QL_IDLE_PROCESSING_TIMER && + (doPark = LENGTH_UPDATER.compareAndSet(this, QL_IDLE_PROCESSING_TIMER, QL_IDLE)) + && taskQueue.peek() == null) { + UA.park0(false, nanos < 0 ? 0 : nanos); + } + LENGTH_UPDATER.addAndGet(this, doPark ? -QL_IDLE : -QL_IDLE_PROCESSING_TIMER); + } + } + } + + @Override + public void run() { + try { + Wisp2Engine engine = (Wisp2Engine) WispEngine.current(); + engine.group = group; + engine.carrier = this; + group.carrierEngines.add(engine); + runCarrier(engine); + } finally { + WispEngine.carrierThreads.remove(thread); + } + } + + boolean idle() { + return queueLength == QL_IDLE; + } + + Runnable pollTask(boolean isSteal) { + StealAwareRunnable task = taskQueue.poll(); + if (task != null) { + if (isSteal && !task.isStealEnable()) { + // disable steal is a very uncommon case, + // The overhead here is acceptable + taskQueue.offer(task); + return null; + } + LENGTH_UPDATER.decrementAndGet(this); + } + return task; + } + + /** + * @return if it is idle + */ + boolean pushAndSignal(StealAwareRunnable task) { + taskQueue.offer(task); + return doSignalIfNecessary(LENGTH_UPDATER.getAndIncrement(this)); + } + + void signal() { + doSignalIfNecessary(queueLength); + } + + /** + * @return if it is idle + */ + private boolean doSignalIfNecessary(int len) { + if (len == QL_IDLE) { + UA.unpark0(thread); + } + return len < 0; + } + + Wisp2Scheduler theScheduler() { + return Wisp2Scheduler.this; + } + } + + /** + * try steal one task from the most busy carrier's queue + * + * @param n max retry times + * @param r random seed + */ + private Runnable trySteal(final int n, int r) { + Carrier busyCarrier = null; + for (int i = 0; i < n; i++) { + final Carrier c = getCarrier(r + i); + int ql = c.queueLength; + if (ql > STEAL_HIGH_WATER_LEVEL) { + Runnable task = c.pollTask(true); + if (task != null) { + return task; + } + } + if (busyCarrier == null || ql > busyCarrier.queueLength) { + busyCarrier = c; + } + } + // If busyCarrier's head is disable steal, we also miss the + // chance of steal from second busy carrier.. + // For disable steal is uncommon path, current implementation is good enough.. + return busyCarrier == null ? null : busyCarrier.pollTask(true); + } + + /** + * Find an idle work, and push task to it's work queue + * + * @param n retry times + * @param ignoreIfBusy ignore this push operation if all carrier if busy + * @param needEnqueue also do enqueue + * @param command the task + */ + private void findIdleAndWakeup(final int n, boolean ignoreIfBusy, + boolean needEnqueue, StealAwareRunnable command) { + int r = (seed = nextRandom(seed)); + Carrier idleCarrier = null; + for (int i = 0; i < n; i++) { + final Carrier c = getCarrier(r + i); + if (c.idle()) { + if (needEnqueue) { + c.pushAndSignal(command); + } else { + c.signal(); + } + return; + } else if (idleCarrier == null || c.queueLength < idleCarrier.queueLength) { + idleCarrier = c; + } + } + if (!ignoreIfBusy && idleCarrier != null) { + if (needEnqueue) { + idleCarrier.pushAndSignal(command); + } else { + idleCarrier.signal(); + } + } + } + + private Carrier getCarrier(int r) { + return carriers[(r & IDX_MASK) % PARALLEL]; + } + + private Carrier castToCarrier(Thread thread) { + if (thread == null) { + return null; + } + Wisp2Engine engine = (Wisp2Engine) JLA.getWispEngine(thread); + return engine.carrier != null && engine.carrier.theScheduler() == this ? + engine.carrier : null; + } + + void addTimer(TimeOut timeOut, Thread current) { + Carrier carrier = castToCarrier(current); + if (carrier != null) { + carrier.timerManager.addTimer(timeOut); + } else { + findIdleAndWakeup(1, false, true, + new StealAwareRunnable() { + @Override + public void run() { + Carrier carrier = castToCarrier(JLA.currentThread0()); + assert carrier != null; + carrier.timerManager.addTimer(timeOut); + } + }); + } + } + + void cancelTimer(TimeOut timeOut, Thread current) { + Carrier carrier = castToCarrier(current); + if (carrier != null) { + carrier.timerManager.cancelTimer(timeOut); + } + } + + /** + * Run the command on the specified thread. + * Used to implement Thread affinity for scheduler. + * + * @param command the code + * @param thread target thread + */ + void executeWithCarrierThread(StealAwareRunnable command, Thread thread) { + final Carrier carrier = castToCarrier(thread); + if (carrier != null && !carrier.detached) { + if (!carrier.pushAndSignal(command)) { + // carrier is busy, wakeup a idle carrier to steal task + if (HELP_STEAL_RETRY > 0) { + findIdleAndWakeup(HELP_STEAL_RETRY, true, false, command); + } + } + } else { + execute(command); + } + } + + /** + * Executes the given command at some time in the future. + * + * @param command the runnable task + * @throws NullPointerException if command is null + */ + public void execute(StealAwareRunnable command) { + findIdleAndWakeup(PUSH_RETRY, false, true, command); + } + + /** + * Detach carrier and create a new carrier to replace it. + */ + void handOffCarrierThread(Thread thread) { + Carrier carrier = castToCarrier(thread); + if (carrier != null && !carrier.detached) { + carrier.detached = true; + carrier.pushAndSignal(new StealAwareRunnable() { + @Override + public void run() { + } + }); // ensure `detached` visibility + carrier.thread.setName(carrier.thread.getName() + " (HandOff)"); + Carrier[] cs = Arrays.copyOf(this.carriers, carriers.length); + for (int i = 0; i < PARALLEL; i++) { + if (cs[i] == carrier) { + cs[i] = new Carrier(); + cs[i].thread.start(); + break; + } + } + carriers = cs; + Wisp2Engine engine = (Wisp2Engine) JLA.getWispEngine(thread); + } + } + + private static void doExec(Runnable task) { + try { + task.run(); + } catch (Throwable t) { + t.printStackTrace(); + } + } + + private int seed = (int) System.nanoTime(); + + private static int nextRandom(int r) { + r ^= r << 13; + r ^= r >>> 17; + return r ^ (r << 5); + } + + private static final AtomicIntegerFieldUpdater LENGTH_UPDATER = + AtomicIntegerFieldUpdater.newUpdater(Carrier.class, "queueLength"); + private static final UnsafeAccess UA = SharedSecrets.getUnsafeAccess(); + private static final JavaLangAccess JLA = SharedSecrets.getJavaLangAccess(); +} diff --git a/src/java.base/share/classes/com/alibaba/wisp/engine/WispConfiguration.java b/src/java.base/share/classes/com/alibaba/wisp/engine/WispConfiguration.java new file mode 100644 index 00000000000..3aa4fb89c0f --- /dev/null +++ b/src/java.base/share/classes/com/alibaba/wisp/engine/WispConfiguration.java @@ -0,0 +1,247 @@ +package com.alibaba.wisp.engine; + +import sun.security.action.GetPropertyAction; + +import java.io.*; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Properties; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; + +@SuppressWarnings("removal") +class WispConfiguration { + private static final String DELIMITER = ";"; + + static final boolean TRANSPARENT_WISP_SWITCH; + static final boolean ENABLE_THREAD_AS_WISP; + static final boolean ALL_THREAD_AS_WISP; + static final boolean USE_THREAD_POOL_LIMIT; + + static final int STACK_SIZE; + static final boolean PARK_ONE_MS_AT_LEAST; + // Limit number of submitted tasks under threshold + static final int EXTERNAL_SUBMIT_THRESHOLD; + static final boolean GLOBAL_POLLER; + static final int WISP_VERSION; + static final int WORKER_COUNT; + static final boolean ENABLE_HANDOFF; + static final int SYSMON_TICK_US; + // monitor + static final boolean WISP_PROFILE_DETAIL; + static final boolean PERF_LOG_ENABLED; + static final int PERF_LOG_INTERVAL_MS; + static final String PERF_LOG_PATH; + + // wisp2 + static final boolean WISP_HIGH_PRECISION_TIMER; + static final boolean WISP_DAEMON_WORKER; + static final boolean WISP_USE_STEAL_LOCK; + static final int WISP_ENGINE_TASK_CACHE_SIZE; + static final int WISP_SCHEDULE_STEAL_RETRY; + static final int WISP_SCHEDULE_PUSH_RETRY; + static final int WISP_SCHEDULE_HELP_STEAL_RETRY; + + + private static List PROXY_SELECTOR_STACK_LIST; + private static List SLEEP_SELECTOR_STACK_LIST; + private static List PUT_TO_CURRENT_STACK_LIST; + private static List PUT_TO_MANAGED_THREAD_STACK_LIST; + private static List PUT_STACK_BLACKLIST; + + static { + Properties p = java.security.AccessController.doPrivileged( + new java.security.PrivilegedAction() { + public Properties run() { + return System.getProperties(); + } + } + ); + + final int CORES = Runtime.getRuntime().availableProcessors(); + TRANSPARENT_WISP_SWITCH = p.containsKey("com.alibaba.wisp.transparentWispSwitch") ? + parseBooleanParameter(p, "com.alibaba.wisp.transparentWispSwitch", false) : + parseBooleanParameter(p, "com.alibaba.transparentAsync", false); + ENABLE_THREAD_AS_WISP = p.containsKey("com.alibaba.wisp.enableThreadAsWisp") ? + parseBooleanParameter(p, "com.alibaba.wisp.enableThreadAsWisp", false) : + parseBooleanParameter(p, "com.alibaba.shiftThreadModel", false); + ALL_THREAD_AS_WISP = parseBooleanParameter(p, "com.alibaba.wisp.allThreadAsWisp", false); + STACK_SIZE = parsePositiveIntegerParameter(p, "com.alibaba.wisp.stacksize", 512 * 1024); + EXTERNAL_SUBMIT_THRESHOLD = parsePositiveIntegerParameter(p, "com.alibaba.wisp.limit", 100); + PARK_ONE_MS_AT_LEAST = parseBooleanParameter(p, "com.alibaba.wisp.parkOneMs", true); + USE_THREAD_POOL_LIMIT = parseBooleanParameter(p, "com.alibaba.threadPoolLimit", true); + GLOBAL_POLLER = parseBooleanParameter(p, "com.alibaba.globalPoller", true); + WISP_VERSION = parsePositiveIntegerParameter(p, "com.alibaba.wisp.version", 1); + WORKER_COUNT = parsePositiveIntegerParameter(p, "com.alibaba.wisp.carrierEngines", CORES); + ENABLE_HANDOFF = parseBooleanParameter(p, "com.alibaba.wisp.enableHandOff", false); + SYSMON_TICK_US = parsePositiveIntegerParameter(p, "com.alibaba.wisp.sysmonTickUs", + (int) TimeUnit.SECONDS.toMicros(1)); + PERF_LOG_ENABLED = parseBooleanParameter(p, "com.alibaba.wisp.enablePerfLog", false); + PERF_LOG_INTERVAL_MS = parsePositiveIntegerParameter(p, "com.alibaba.wisp.logTimeInternalMillis", 15000); + if (PERF_LOG_ENABLED) { + WISP_PROFILE_DETAIL = true; + PERF_LOG_PATH = p.getProperty("com.alibaba.wisp.logPath"); + } else { + WISP_PROFILE_DETAIL = parseBooleanParameter(p, "com.alibaba.wisp.profileDetail", false); + PERF_LOG_PATH = ""; + } + + WISP_HIGH_PRECISION_TIMER = parseBooleanParameter(p, "com.alibaba.wisp.highPrecisionTimer", false); + WISP_DAEMON_WORKER = parseBooleanParameter(p, "com.alibaba.wisp.daemonWorker", true); + WISP_USE_STEAL_LOCK = parseBooleanParameter(p, "com.alibaba.wisp.useStealLock", true); + WISP_ENGINE_TASK_CACHE_SIZE = parsePositiveIntegerParameter(p, "com.alibaba.wisp.engineTaskCache", 20); + WISP_SCHEDULE_STEAL_RETRY = parsePositiveIntegerParameter(p, "com.alibaba.wisp.schedule.stealRetry", CORES); + WISP_SCHEDULE_PUSH_RETRY = parsePositiveIntegerParameter(p, "com.alibaba.wisp.schedule.pushRetry", CORES); + WISP_SCHEDULE_HELP_STEAL_RETRY = parsePositiveIntegerParameter(p, "com.alibaba.wisp.schedule.helpStealRetry", + Math.max(1, CORES / 2)); + checkCompatibility(); + } + + private static void checkCompatibility() { + if (WISP_VERSION != 1 && WISP_VERSION != 2) { + throw new IllegalArgumentException("\"-Dcom.alibaba.wisp.version=\" is only allowed to be 1 or 2"); + } + checkDependency(ENABLE_THREAD_AS_WISP, "-Dcom.alibaba.wisp.enableThreadAsWisp=true", + TRANSPARENT_WISP_SWITCH, "-Dcom.alibaba.wisp.transparentWispSwitch=true"); + checkDependency(ENABLE_HANDOFF, "-Dcom.alibaba.wisp.enableHandOff=true", + TRANSPARENT_WISP_SWITCH, "-Dcom.alibaba.wisp.enableThreadAsWisp=true"); + checkDependency(WISP_VERSION == 2, "-Dcom.alibaba.wisp.version=2", + GLOBAL_POLLER, "-Dcom.alibaba.globalPoller=true"); + checkDependency(ALL_THREAD_AS_WISP, "-Dcom.alibaba.wisp.allThreadAsWisp=true", + ENABLE_THREAD_AS_WISP && WISP_VERSION == 2, + "-Dcom.alibaba.wisp.enableThreadAsWisp=true -Dcom.alibaba.wisp.version=2"); + } + + private static void checkDependency(boolean cond, String condStr, boolean preRequire, String preRequireStr) { + if (cond && !preRequire) { + throw new IllegalArgumentException("\"" + condStr + "\" depends on \"" + preRequireStr + "\""); + } + } + + private static int parsePositiveIntegerParameter(Properties p, String key, int defaultVal) { + String value; + if (p == null || (value = p.getProperty(key)) == null) { + return defaultVal; + } + int res = defaultVal; + try { + res = Integer.valueOf(value); + } catch (NumberFormatException e) { + return defaultVal; + } + return res <= 0 ? defaultVal : res; + } + + private static boolean parseBooleanParameter(Properties p, String key, boolean defaultVal) { + String value; + if (p == null || (value = p.getProperty(key)) == null) { + return defaultVal; + } + return Boolean.valueOf(value); + } + + private static List parseListParameter(Properties p, String key) { + String value = p.getProperty(key); + return value == null ? Collections.emptyList() : + Arrays.asList(value.trim().split(DELIMITER)); + } + + /** + * Loading config from system property "com.alibaba.wisp.config" specified + * file or jre/lib/wisp.properties. + */ + @SuppressWarnings("removal") + private static void loadBizConfig() { + String path = java.security.AccessController.doPrivileged( + new GetPropertyAction("com.alibaba.wisp.config")); + if (path == null || !new File(path).isFile()) { + path = java.security.AccessController.doPrivileged( + new GetPropertyAction("java.home")) + + File.separator + "conf" + File.separator + "wisp.properties"; + } + + File f = new File(path); + if (f.exists()) { + Properties p = new Properties(); + try (InputStream is = new BufferedInputStream(new FileInputStream(f.getPath()))) { + p.load(is); + } catch (IOException e) { + // ignore, all STACK_LIST are empty + } + PROXY_SELECTOR_STACK_LIST = parseListParameter(p, "com.alibaba.wisp.biz.selector"); + PUT_TO_CURRENT_STACK_LIST = parseListParameter(p, "com.alibaba.wisp.biz.current"); + PUT_TO_MANAGED_THREAD_STACK_LIST = parseListParameter(p, "com.alibaba.wisp.biz.manage"); + PUT_STACK_BLACKLIST = parseListParameter(p, "com.alibaba.wisp.biz.black"); + SLEEP_SELECTOR_STACK_LIST = parseListParameter(p, "com.alibaba.wisp.biz.selector.sleep"); + } + } + + private static final int UNLOADED = 0, LOADING = 1, LOADED = 2; + private static AtomicInteger bizLoadStatus = new AtomicInteger(UNLOADED); + + private static void ensureBizConfigLoaded() { + if (bizLoadStatus.get() == LOADED) { + return; + } + if (bizLoadStatus.get() == UNLOADED && bizLoadStatus.compareAndSet(UNLOADED, LOADING)) { + try { + loadBizConfig(); + } finally { + bizLoadStatus.set(LOADED); + } + } + while (bizLoadStatus.get() != LOADED) {/* wait */} + } + + static boolean ifPutToCurrentEngine() { + if (ALL_THREAD_AS_WISP) { + return false; + } + ensureBizConfigLoaded(); + StackTraceElement bt[] = Thread.currentThread().getStackTrace(); + return !currentStackContains(PUT_STACK_BLACKLIST, bt) && + currentStackContains(PUT_TO_CURRENT_STACK_LIST, bt); + } + + static boolean ifProxySelector() { + if (ALL_THREAD_AS_WISP) { + return false; + } + ensureBizConfigLoaded(); + return currentStackContains(PROXY_SELECTOR_STACK_LIST, Thread.currentThread().getStackTrace()); + } + + static boolean ifSpinSelector() { + if (ALL_THREAD_AS_WISP) { + return false; + } + ensureBizConfigLoaded(); + return currentStackContains(SLEEP_SELECTOR_STACK_LIST, Thread.currentThread().getStackTrace()); + } + + static boolean ifPutToManagedThread() { + if (ALL_THREAD_AS_WISP) { + return false; + } + ensureBizConfigLoaded(); + StackTraceElement bt[] = Thread.currentThread().getStackTrace(); + for (StackTraceElement aBt : bt) { + if (aBt.getClassName().equals(WispWorkerContainer.class.getName())) return false; + } + return !currentStackContains(PUT_STACK_BLACKLIST, bt) && + currentStackContains(PUT_TO_MANAGED_THREAD_STACK_LIST, bt); + } + + private static boolean currentStackContains(List stacks, StackTraceElement bt[]) { + for (String clazzAndMethod : stacks) { + for (StackTraceElement aBt : bt) { + if (aBt.getClassName().length() + aBt.getMethodName().length() + 2 == clazzAndMethod.length() && + clazzAndMethod.startsWith(aBt.getClassName()) && clazzAndMethod.endsWith(aBt.getMethodName()) && + clazzAndMethod.regionMatches(aBt.getClassName().length(), "::", 0, 2)) + return true; + } + } + return false; + } +} diff --git a/src/java.base/share/classes/com/alibaba/wisp/engine/WispCounter.java b/src/java.base/share/classes/com/alibaba/wisp/engine/WispCounter.java new file mode 100644 index 00000000000..7b36b0d0a2e --- /dev/null +++ b/src/java.base/share/classes/com/alibaba/wisp/engine/WispCounter.java @@ -0,0 +1,222 @@ +package com.alibaba.wisp.engine; + +final class WispCounter { + + private long switchCount = 0; + + private long waitTimeTotal = 0; + + private long runningTimeTotal = 0; + + private long completeTaskCount = 0; + + private long createTaskCount = 0; + + private long parkCount = 0; + + private long unparkCount = 0; + + private long unparkInterruptSelectorCount = 0; + + private long selectableIOCount = 0; + + private long timeOutCount = 0; + + private long eventLoopCount = 0; + + private long totalEnqueueTime = 0; + + private long enqueueCount = 0; + + private long totalExecutionTime = 0; + + private long executionCount = 0; + + private long totalWaitSocketIOTime = 0; + + private long waitSocketIOCount = 0; + + private long totalBlockingTime = 0; + + private long unparkFromJvmCount = 0; + + WispEngine engine; + + private WispCounter(WispEngine engine) { + this.engine = engine; + } + + boolean getRunningState() { + return engine.isRunning(); + } + + void incrementSwitchCount() { + switchCount++; + } + + long getSwitchCount() { + return switchCount; + } + + void incrementCompleteTaskCount() { + completeTaskCount++; + } + + void incrementRunningTimeTotal(long value) { + runningTimeTotal += value; + } + + long getRunningTimeTotal() { + return runningTimeTotal; + } + + void incrementWaitTime(long value) { + waitTimeTotal += value; + } + + long getWaitTimeTotal() { + return waitTimeTotal; + } + + long getCompleteTaskCount() { + return completeTaskCount; + } + + void incrementCreateTaskCount() { + createTaskCount++; + } + + long getCreateTaskCount() { + return createTaskCount; + } + + void incrementParkCount() { + parkCount++; + } + + long getParkCount() { + return parkCount; + } + + + void incrementUnparkInterruptSelectorCount() { + unparkInterruptSelectorCount++; + } + + long getUnparkInterruptSelectorCount() { + return unparkInterruptSelectorCount; + } + + void incrementSelectableIOCount() { + selectableIOCount++; + } + + long getSelectableIOCount() { + return selectableIOCount; + } + + void incrementTimeOutCount() { + timeOutCount++; + } + + long getTimeOutCount() { + return timeOutCount; + } + + void incrementEventLoopCount() { + eventLoopCount++; + } + + long getEventLoopCount() { + return eventLoopCount; + } + + void incrementTotalEnqueueTime(long value) { + totalEnqueueTime += value; + enqueueCount++; + } + + long getTotalEnqueueTime() { + return totalEnqueueTime; + } + + long getEnqueueCount() { + return enqueueCount; + } + + void incrementTotalExecutionTime(long value) { + totalExecutionTime += value; + executionCount++; + } + + long getTotalExecutionTime() { + return totalExecutionTime; + } + + long getExecutionCount() { + return executionCount; + } + + void incrementTotalWaitSocketIOTime(long value) { + totalWaitSocketIOTime += value; + waitSocketIOCount++; + } + + long getTotalWaitSocketIOTime() { + return totalWaitSocketIOTime; + } + + long getWaitSocketIOCount() { + return waitSocketIOCount; + } + + void incrementTotalBlockingTime(long value) { + totalBlockingTime += value; + unparkCount++; + } + + long getTotalBlockingTime() { + return totalBlockingTime; + } + + long getUnparkCount() { + return unparkCount; + } + + public long getQueueLength() { + return engine.getTaskQueueLength(); + } + + long getNumberOfRunningTasks() { + return engine.getNumberOfRunningTasks(); + } + + void incrementUnparkFromJvmCount() { + unparkFromJvmCount++; + } + + long getUnparkFromJvmCount() { + return unparkFromJvmCount; + } + + WispCounter() {} + + void assign(WispCounter counter) { + createTaskCount = counter.createTaskCount; + completeTaskCount = counter.completeTaskCount; + totalEnqueueTime = counter.totalEnqueueTime; + enqueueCount = counter.enqueueCount; + totalExecutionTime = counter.totalExecutionTime; + executionCount = counter.executionCount; + totalBlockingTime = counter.totalBlockingTime; + unparkCount = counter.unparkCount; + totalWaitSocketIOTime = counter.totalWaitSocketIOTime; + waitSocketIOCount = counter.waitSocketIOCount; + switchCount = counter.switchCount; + unparkFromJvmCount = counter.unparkFromJvmCount; + } + + public static WispCounter create(WispEngine engine) { + return new WispCounter(engine); + } +} diff --git a/src/java.base/share/classes/com/alibaba/wisp/engine/WispEngine.java b/src/java.base/share/classes/com/alibaba/wisp/engine/WispEngine.java new file mode 100644 index 00000000000..e554cf2dbc7 --- /dev/null +++ b/src/java.base/share/classes/com/alibaba/wisp/engine/WispEngine.java @@ -0,0 +1,944 @@ +package com.alibaba.wisp.engine; + +import jdk.internal.access.JavaLangAccess; +import jdk.internal.access.SharedSecrets; +import jdk.internal.access.WispEngineAccess; +import jdk.internal.misc.Unsafe; + +import java.dyn.CoroutineExitException; +import java.dyn.CoroutineSupport; +import java.io.IOException; +import java.nio.channels.SelectableChannel; +import java.nio.channels.Selector; +import java.text.SimpleDateFormat; +import java.util.*; +import java.util.concurrent.*; + +/** + * Coroutine Runtime Engine. It's a "wisp" thing, as we want our asynchronization transformation to be transparent + * without asking a Java programmer to modify his/her code. + *

+ *

{@link WispTask} provides high-level semantics of {link @Coroutine}. + * {@link WispTask} is created, managed and scheduled by {@link WispEngine}. + *

+ *

A {@link WispEngine} instance is expected to run in a specific thread. Get per-thread instance by calling + * {@link WispEngine#current()}. + */ +public abstract class WispEngine extends AbstractExecutorService { + + /* + some of our users change this field by reflection + in the runtime to disable wisp temporarily. + We should move shiftThreadModel to WispConfiguration + after we provide api to control this behavior and + notify the users to modify their code. + + TODO refactor to com.alibaba.wisp.enableThreadAsWisp later + */ + private static boolean shiftThreadModel; + + public static boolean transparentWispSwitch() { + return WispConfiguration.TRANSPARENT_WISP_SWITCH; + } + + public static boolean enableThreadAsWisp() { + return shiftThreadModel; + } + + @Deprecated + public static boolean isTransparentAsync() { + return transparentWispSwitch(); + } + + @Deprecated + public static boolean isShiftThreadModel() { + return shiftThreadModel; + } + + private static final Unsafe UNSAFE = Unsafe.getUnsafe(); + static final JavaLangAccess JLA = SharedSecrets.getJavaLangAccess(); + /* + * Wisp specified Thread Group + * all the daemon threads in wisp should be created with the Thread Group. + * In Thread.start(), if the thread should not convert to WispTask, + * check whether the thread's group is daemonThreadGroup + */ + static ThreadGroup daemonThreadGroup; + static Set carrierThreads; + + static { + registerNatives(); + setWispEngineAccess(); + } + + private static void initializeWispClass() { + assert JLA != null : "WispEngine should be initialized after System"; + assert JLA.currentThread0().getName().equals("main") : "Wisp need to be loaded by main thread"; + shiftThreadModel = WispConfiguration.ENABLE_THREAD_AS_WISP; + daemonThreadGroup = new ThreadGroup(JLA.currentThread0().getThreadGroup(), "Daemon Thread Group"); + + carrierThreads = new ConcurrentSkipListSet<>(new Comparator() { + @Override + public int compare(Thread o1, Thread o2) { + return Long.compare(o1.getId(), o2.getId()); + } + }); + if (transparentWispSwitch()) { + initializeClasses(); + JLA.wispBooted(); + } + } + + private static void startWispDaemons() { + if (transparentWispSwitch()) { + Thread unparker = new Thread(daemonThreadGroup, new Runnable() { + @Override + public void run() { + int[] proxyUnparks = new int[12]; + CoroutineSupport.setWispBooted(); + while (true) { + int n = getProxyUnpark(proxyUnparks); + for (int i = 0; i < n; i++) { + WispTask task = WispTask.fromId(proxyUnparks[i]); + if (task != null) { + task.unpark(); + } // else: target engine exited + } + } + } + }, "Wisp-Unpark-Dispatcher"); + unparker.setDaemon(true); + unparker.start(); + WispSysmon.INSTANCE.startDaemon(); + if (WispConfiguration.WISP_VERSION == 2) { + Wisp2Group.WISP2_ROOT_GROUP.scheduler.startCarrierThreads(); + } + // TODO: move poller thread start in future patch + } + } + + private static void setWispEngineAccess() { + SharedSecrets.setWispEngineAccess(new WispEngineAccess() { + @Override + public WispTask getCurrentTask() { + return WispEngine.current().getCurrentTask(); + } + + @Override + public void dispatch(Runnable runnable, String name) { + WispEngine engine = current(); + engine.dispatchTask(runnable, name); + } + + @Override + public void eventLoop() { + WispEngine.eventLoop(); + } + + @Override + public void registerEvent(SelectableChannel ch, int events) throws IOException { + WispEngine.current().registerEvent(ch, events); + } + + @Override + public void registerEpollEvent(int epFd) throws IOException { + assert WispConfiguration.WISP_VERSION == 2; + WispEngine.current().registerEpollEvent(epFd); + } + + @Override + public boolean usingWispEpoll(Thread t) { + return WispConfiguration.WISP_VERSION == 2 && runningAsCoroutine(t); + } + + @Override + public void unregisterEvent() { + WispEngine.current().unregisterEvent(); + } + + @Override + public void yieldOnBlocking() { + WispEngine.current().yieldOnBlocking(); + } + + @Override + public void addTimer(long deadlineNano) { + WispEngine.current().addTimer(deadlineNano, false); + } + + @Override + public void cancelTimer() { + WispEngine.current().cancelTimer(); + } + + @Override + public void sleep(long ms) { + WispTask.sleep(ms); + } + + @Override + public void yield() { + WispEngine.current().yield(); + } + + @Override + public boolean isThreadTask(WispTask task) { + return task.isThreadTask(); + } + + @Override + public boolean isTimeout() { + WispTask task = WispEngine.current().current; + return task.timeOut != null && task.timeOut.expired(); + } + + @Override + public void park(long timeoutNano) { + WispTask.jdkPark(timeoutNano); + } + + @Override + public void unpark(WispTask task) { + if (task != null) { + task.jdkUnpark(); + } + } + + @Override + public void destroy() { + WispEngine.current().destroy(); + } + + @Override + public T runInCritical(CheckedSupplier supplier) { + WispEngine engine = null; + boolean critical0 = false; + if (WispEngine.transparentWispSwitch()) { + engine = WispEngine.current(); + critical0 = engine.isInCritical; + engine.isInCritical = true; + } + try { + return supplier.get(); + } catch (Throwable t) { + UNSAFE.throwException(t); + return null; // make compiler happy + } finally { + if (engine != null) { + engine.isInCritical = critical0; + } + } + } + + @Override + public boolean hasMoreTasks() { + return WispEngine.current().getTaskQueueLength() > 0; + } + + @Override + public boolean runningAsCoroutine(Thread t) { + return WispEngine.runningAsCoroutine(t); + } + + public boolean isAlive(WispTask task) { + return task.isAlive(); + } + + @Override + public void interrupt(WispTask task) { + task.interrupt(); + } + + @Override + public boolean isInterrupted(WispTask task) { + return task.isInterrupted(); + } + + @Override + public boolean testInterruptedAndClear(WispTask task, boolean clear) { + return task.testInterruptedAndClear(clear); + } + + @Override + public boolean ifPutToCurrentEngine() { + return WispConfiguration.ifPutToCurrentEngine(); + } + + @Override + public boolean ifProxySelector() { + return WispConfiguration.ifProxySelector(); + } + + @Override + public boolean ifSpinSelector() { + return WispConfiguration.ifSpinSelector(); + } + + @Override + public boolean ifPutToManagedThread() { + return WispConfiguration.ifPutToManagedThread(); + } + + @Override + public boolean useThreadPoolLimit() { + return WispConfiguration.USE_THREAD_POOL_LIMIT; + } + + @Override + public String getThreadUsage(String threadName) { + return WispWorkerContainer.getThreadUsage(threadName); + } + + private final static String JAVA_LANG = "java.lang"; + + private final static String JDK_INTERNAL = "jdk.internal"; + + /** + * Try to "start thread" as wisp if all listed condition is satisfied: + * + * 1. not in java.lang package + * 2. not a WispEngine Thread (may overlapping 2; except user created wisp2 carrier) + * 3. allThreadAsWisp is true or call stack satisfies wisp.conf's description. + * + * @param thread the thread + * @param target thread's target field + * + * @return if condition is satisfied and thread is started as wisp + */ + @Override + public boolean tryStartThreadAsWisp(Thread thread, Runnable target) { + if (thread.getClass().getName().startsWith(JAVA_LANG) && + (target == null || target.getClass().getName().startsWith(JAVA_LANG))) { + return false; + } + if (thread.getClass().getName().startsWith(JDK_INTERNAL) && + (target == null || target.getClass().getName().startsWith(JDK_INTERNAL))) { + return false; + } + if (WispEngine.isEngineThread(thread) || + !(WispConfiguration.ALL_THREAD_AS_WISP || WispConfiguration.ifPutToManagedThread())) { + return false; + } + // pthread_create always return before new thread started, so we should not wait here + JLA.setWispAlive(thread, true); // thread.isAlive() should be true + WispWorkerContainer.INSTANCE.dispatch(getThreadUsage(thread.getName()), thread.getName(), thread, thread); + return true; + } + }); + } + + private static void initializeClasses() { + try { + Class.forName(CoroutineExitException.class.getName()); + Class.forName(WispThreadWrapper.class.getName()); + if (WispConfiguration.GLOBAL_POLLER) { + Class.forName(WispPoller.class.getName()); + } + WispEngine.current().preloadClasses(); + } catch (Exception e) { + throw new ExceptionInInitializerError(e); + } + } + + private static boolean isEngineThread(Thread t) { + assert daemonThreadGroup != null; + return daemonThreadGroup == t.getThreadGroup() || carrierThreads.contains(t); + } + + static boolean runningAsCoroutine(Thread t) { + if (WispConfiguration.WISP_VERSION == 1) { + // For wisp1: + // Any thread could create WispTask, we need + // hook all blocking API to avoid WispTask blocking each other + return true; + } + assert WispConfiguration.WISP_VERSION == 2; + + WispTask task = t == null ? WispEngine.current().getCurrentTask() : JLA.getWispTask(t); + assert task != null; + // For wisp2: + // Only carrierThread could create WispTask, and + // the carrierThread will listen on WispTask's wakeup. + // So we can safely letting the threadTask block the whore Thread. + return !task.isThreadTask(); + } + + WispEngine() { + thread = JLA.currentThread0(); + CoroutineSupport cs = thread.getCoroutineSupport(); + current = threadTask = new WispTask(this, + cs == null ? null : cs.threadCoroutine(), + cs != null, true); + if (cs == null) { // fake engine used in jni attach + threadTask.setThreadWrapper(thread); + } else { + threadTask.reset(null, null, + "THREAD: " + thread.getName(), thread, thread.getContextClassLoader()); + } + } + + /** + * Use 2nd-phase init after constructor. Because if constructor calls Thread.currentThread(), + * and recursive calls constructor, then stackOverflow. + */ + private void init() { + WispTask.trackTask(threadTask); + counter = WispCounter.create(this); + } + + /** + * Constructor is private, so one can only get thread-specific engine by calling this method. + * This method's name indicates a task is created in current thread's engine. + *

+ * We can not use ThreadLocal any more, because if transparentAsync, it behaves as a coroutine local. + * + * @return thread-specific engine + */ + public static WispEngine current() { + Thread thread = JLA.currentThread0(); + WispEngine engine = JLA.getWispEngine(thread); + if (engine == null) { + engine = WispConfiguration.WISP_VERSION == 2 ? + new Wisp2Engine(Wisp2Group.WISP2_ROOT_GROUP) : + new ScheduledWispEngine(); + if (engine.threadTask.ctx != null) { + JLA.setWispEngine(thread, engine); + engine.init(); + engine.postInit(); + } // else: fake engine used in jni attach + } + return engine; + } + + protected final Thread thread; + // current running task + protected WispTask current; + protected final WispTask threadTask; + protected Set runningTasks = new HashSet<>(); + protected List taskCache = new ArrayList<>(); + + private int createdTasks; + + protected boolean isInCritical; + private boolean hasBeenShutdown; + // do not need volatile, because we do force enqueue after + // `hasBeenShutdown` becomes true, the flag could always be seen + + Statistics statistics = new Statistics(); + + WispCounter counter; + int schedTick; + int lastSchedTick; // access by Sysmon + boolean terminated; + private long switchTimestamp = 0; + + /** + * Entry point for running an engine. + * Please make sure calling {@code WispTask.createTask(...)} somewhere, so WispTasks were created before call + * {@code eventLoop()} + *

+ * A typical server looks like this: + * 

+     *  WispEngine.dispatch(() -> {
+     *      while (client = accept()) {
+     *          WispEngine.dispatch(
+     *                  () -> handle(client), "client handler");
+     *      }
+     *  }, "accepter");
+     *
+     *  WispEngine.eventLoop(); // loop forever
+     * 
+     */
+    private static void eventLoop() {
+        WispEngine engine = current();
+        while (!engine.runningTasks.isEmpty()) {
+            engine.yieldToNext();
+        }
+    }
+
+    /**
+     * @return Currently running WispTask. Ensured by {@link #yieldTo(WispTask)}
+     * If calling in a non-coroutine environment, return a thread-emulated WispTask.
+     */
+    WispTask getCurrentTask() {
+        return current;
+    }
+
+    WispCounter getCounter() {
+        return counter;
+    }
+
+    /**
+     * Create WispTask to run task code
+     * 

+     * The real running thread depends on implementation
+     *
+     * @param target target code
+     */
+    public static void dispatch(Runnable target) {
+        WispEngine engine = current();
+        engine.dispatchTask(target, "dispatch task");
+    }
+
+    /**
+     * Submit a WispTask in this engine from outside thread(engine).
+     * 

+     * In this way, we created 2 WispTasks inefficiently.
+     * Use this method only if you need the reference of created task,
+     * else {@link #execute(Runnable)}'s semantic is enough.
+     *
+     * @param target the code
+     * @param name   coroutine's name
+     * @param thread the value returned by {@link Thread#currentThread()}
+     *               could be null
+     */
+    Future summitTask(Runnable target, String name, Thread thread) {
+        return submit(() -> WispEngine.current().runTaskInternal(target, name, thread, current.ctxClassLoader));
+    }
+
+    final WispTask runTaskInternal(Runnable target, String name, Thread thread, ClassLoader ctxLoader) {
+        if (hasBeenShutdown) {
+            return null;
+        }
+        boolean isInCritical0 = isInCritical;
+        isInCritical = true;
+        WispTask wispTask;
+        try {
+            if (0 == createdTasks++) {
+                WispSysmon.INSTANCE.register(this);
+            }
+            counter.incrementCreateTaskCount();
+            if ((wispTask = getTaskFromCache()) == null) {
+                wispTask = new WispTask(this, null, true, false);
+                WispTask.trackTask(wispTask);
+            }
+            wispTask.reset(target, current, name, thread, ctxLoader);
+            runningTasks.add(wispTask);
+        } finally {
+            isInCritical = isInCritical0;
+        }
+        yieldTo(wispTask);
+        runWispTaskEpilog();
+
+        return wispTask;
+    }
+
+
+    /**
+     * Wake up a {@link WispTask} that belongs to this engine,
+     * and caller will also do a {@link Selector#wakeup()} if engine's thread is being blocked on Selector.
+     *
+     * @param task target task
+     */
+    boolean wakeupTask(WispTask task) {
+        return wakeupTask(task, false);
+    }
+
+    /**
+     * Block current coroutine and do scheduling.
+     * Typically called when resource is not ready.
+     */
+    protected void yieldOnBlocking() {
+        current.countExecutionTime(switchTimestamp);
+        if (current.status == WispTask.Status.RUNNABLE) {
+            current.status = WispTask.Status.BLOCKED;
+        }
+
+        if (current.parent != null && current.parent.isRunnable()) {
+            WispTask parent = current.parent;
+            assert parent.engine == this;
+            // DESIGN:
+            // only the first park of wisp should go back to parent
+            current.parent = null;
+            yieldTo(parent);
+        } else {
+            yieldToNext();
+        }
+        if (hasBeenShutdown) {
+            doShutdown();
+        }
+    }
+
+    /**
+     * The ONLY entry point to a task,
+     * {@link #current} will be set correctly
+     *
+     * @param task coroutine to run
+     */
+    protected boolean yieldTo(WispTask task) {
+        if (task == null) {
+            return false;
+        }
+        schedTick++;
+
+        assert task.engine == this;
+
+        if (task == current) {
+            task.status = WispTask.Status.RUNNABLE;
+            switchTimestamp = getNanoTimeForProfile();
+            return true;
+        }
+
+        if (task.status == WispTask.Status.BLOCKED)
+            task.status = WispTask.Status.RUNNABLE;
+
+        if (task.status == WispTask.Status.ZOMBIE) {
+            unregisterEvent(task);
+            return false;
+        }
+
+        WispTask from = current;
+        current = task;
+        counter.incrementSwitchCount();
+        assert !isInCritical;
+        WispTask.switchTo(from, task);
+        assert !WispEngine.current().isInCritical;
+        current.engine.switchTimestamp = getNanoTimeForProfile();
+        return true;
+    }
+
+
+    /**
+     * Modify current {@link WispTask}'s interest channel and event.
+     * {@see registerEvent(...)}
+     * 

+     * Used for implementing socket io
+     * 
+     *     while (!ch.read(buf) == 0) { // 0 indicate IO not ready, not EOF..
+     *         registerEvent(ch, OP_READ);
+     *         yieldOnBlocking();
+     *     }
+     *     // read is done here
+     * 
+     */
+    private void registerEvent(SelectableChannel ch, int events) throws IOException {
+        registerEvent(current, ch, events);
+    }
+
+    /**
+     * Clean current task's interest event before an non-IO blocking operation
+     * or task exit to prevent unexpected wake up.
+     */
+    void unregisterEvent() {
+        unregisterEvent(current);
+    }
+
+    private void unregisterEvent(WispTask target) {
+        if (target.ch != null) {
+            try {
+                registerEvent(target, target.ch, 0);
+                target.resetRegisterEventTime();
+                target.ch = null;
+            } catch (IOException e) {
+                // pass
+            }
+        }
+    }
+
+    /**
+     * The only exit path of a task.
+     * WispTask must call {@code taskExit()} to exit safely.
+     */
+    void taskExit() { // and exit
+        current.status = WispTask.Status.ZOMBIE;
+        runningTasks.remove(current);
+
+        current.countExecutionTime(switchTimestamp);
+        switchTimestamp = 0;
+
+        unregisterEvent();
+        returnTaskToCache(current);
+
+        counter.incrementCompleteTaskCount();
+
+        if (runningTasks.isEmpty() && threadTask.isRunnable()) {
+            // finish the event loop
+            yieldTo(threadTask);
+        } else {
+
+            // In Tenant killing process, we have an pending exception,
+            // WispTask.Coroutine's loop will be break
+            // invoke an explicit reschedule instead of return
+            yieldOnBlocking();
+        }
+    }
+
+    private void destroy() {
+        WispTask.cleanExitedTasks(taskCache);
+        WispTask.cleanExitedTask(threadTask);
+        closeEngineSelector();
+        terminated = true;
+    }
+
+
+    // ----------------------------------------------- initialization
+
+    /**
+     * Preload wisp runtime used class.
+     * Only called when WispEngine is loading.
+     */
+    protected abstract void preloadClasses() throws Exception;
+
+    /**
+     * Implementation WispEngine specified initialize.
+     */
+    protected void postInit() {
+    }
+
+
+    // ----------------------------------------------- lifecycle hooks
+
+    /**
+     * hook for leave wispTask entry
+     */
+    protected void runWispTaskEpilog() {
+    }
+
+
+    // ----------------------------------------------- timer related
+
+    /**
+     * Add a timer for current {@link WispTask},
+     * used for implementing timed IO operation / sleep etc...
+     *
+     * @param deadlineNano deadline of the timer
+     * @param fromJvm      synchronized or obj.wait()
+     */
+    protected abstract void addTimer(long deadlineNano, boolean fromJvm);
+
+
+    /**
+     * Cancel the timer added by {@link #addTimer(long, boolean)}.
+     */
+    protected abstract void cancelTimer();
+
+
+    // -----------------------------------------------  yielding
+
+    /**
+     * yield to next runnable task.
+     */
+    protected abstract void yieldToNext();
+
+    /**
+     * Telling to the scheduler that the current thread is willing to yield
+     * its current use of a processor.
+     * 

+     * Called by {@link Thread#yield()}
+     */
+    protected abstract void yield();
+
+
+    // -----------------------------------------------  event related
+
+    /**
+     * register target {@link WispTask}'s interest channel and event.
+     *
+     * @param ch     the channel that is related to the current WispTask
+     * @param events interest event
+     * @throws IOException
+     */
+    protected abstract void registerEvent(WispTask target, SelectableChannel ch, int events) throws IOException;
+
+    /**
+     * register target {@link WispTask}'s interest epoll fd
+     *
+     * @param epFd the epoll fd that is related to the current WispTask
+     * @throws IOException
+     */
+    protected void registerEpollEvent(int epFd) throws IOException {
+        throw new UnsupportedOperationException();
+    }
+
+    // -----------------------------------------------  task related
+
+    /**
+     * Inheritance from {@link AbstractExecutorService}
+     *
+     * @param target the runnable task
+     */
+    public abstract void execute(Runnable target);
+
+    /**
+     * Create a wisp task and run.
+     *
+     * @param target the task to execute
+     * @param name   task name
+     */
+    protected abstract void dispatchTask(Runnable target, String name);
+
+    /**
+     * Wake up a {@link WispTask} that belongs to Wisp Implement,
+     * and caller may need call {@link Selector#wakeup()} if engine's thread is being blocked on Selector.
+     *
+     * @param task   target task
+     * @param urgent give the task a high priority to be schedule, not promised for all implementation
+     */
+    protected abstract boolean wakeupTask(WispTask task, boolean urgent);
+
+    /**
+     * @return task from global cached theScheduler
+     */
+    abstract WispTask getTaskFromCache();
+
+    /**
+     * return task back to global cache
+     */
+    abstract void returnTaskToCache(WispTask task);
+
+    /**
+     * create a Entry runnable for wisp task,
+     * used for bridge coroutine and Executor interface.
+     */
+    StealAwareRunnable createResumeEntry(WispTask task) {
+        return null;
+    }
+
+
+    // -----------------------------------------------  poller
+
+
+    /**
+     * clean resource
+     */
+    protected void closeEngineSelector() {
+    }
+
+    // ----------------------------------------------- status fetch
+
+    /**
+     * @return if current engine is busy
+     */
+    protected abstract boolean isRunning();
+
+    /**
+     * @return queue length. used for mxBean report
+     */
+    protected abstract int getTaskQueueLength();
+
+    /**
+     * @return running task number, used for mxBean report
+     */
+    int getNumberOfRunningTasks() {
+        return runningTasks.size();
+    }
+
+    // -----------------------------------------------  shutdown support
+
+    /**
+     * start up shutdown process, the typical implement is
+     * wake up thread coroutine, then send exception to coroutines
+     * one by one.
+     */
+    protected abstract void startShutdown();
+
+    /**
+     * Send exception to coroutines one by one.
+     */
+    protected abstract void iterateTasksForShutdown();
+
+    // -----------------------------------------------  retake
+
+    /**
+     * hand off wispEngine for blocking system calls.
+     */
+    protected void handOff() {
+    }
+
+    static long getNanoTimeForProfile() {
+        return WispConfiguration.WISP_PROFILE_DETAIL ? System.nanoTime() : 0;
+    }
+
+    void countEnqueueTime(long enqueueTime) {
+        if (enqueueTime != 0) {
+            counter.incrementTotalEnqueueTime(System.nanoTime() - enqueueTime);
+        }
+    }
+
+    long exponentialSmoothed(long accumulator, long newVal) {
+        return (accumulator * 88 + newVal * 12) / 100;
+    }
+
+    class Statistics {
+        int maxRunning;
+        int maxReschedule;
+        int doubleCheckWakeup;
+
+        int eventLoops;
+        int nonBlockSelectCount;
+        int selectNothingActive;
+        int selectNothingPassive;
+        int proxyBlockingSelect;
+        int proxySelectNow;
+        int ioEventCount;
+
+        int doIO;
+        int doWakeup;
+        int doTimer;
+
+        // not precision, because modified by multi-thread
+        int wakeupCount;
+        int realWakeupCount;
+        int innerEngineWakeup;
+        int alreadyWakeup;
+
+        int selectorRebuild;
+        int retryCanceledKey;
+    }
+
+    @Override
+    public String toString() {
+        return new SimpleDateFormat("MM-dd HH:mm:ss.SSS").format(new Date()) + "\n" +
+                "Engine (" + thread.getName() + ") Runtime Info:" +
+                "\nrunningTasks\t" + runningTasks.size() +
+                "\ncreatedTasks\t" + createdTasks +
+                "\neventLoops\t" + statistics.eventLoops +
+                "\nswitchCount\t" + schedTick;
+    }
+
+    @Override
+    public void shutdown() {
+        hasBeenShutdown = true;
+        if (WispEngine.current().current == threadTask) {
+            doShutdown();
+        } else {
+            startShutdown();
+        }
+    }
+
+    private CountDownLatch shutdownFuture = new CountDownLatch(1);
+
+    void doShutdown() {
+        if (current == threadTask) {
+            iterateTasksForShutdown();
+            thread.getCoroutineSupport().drain();
+            shutdownFuture.countDown();
+        } else {
+            UNSAFE.throwException(new ThreadDeath());
+        }
+    }
+
+    @Override
+    public List shutdownNow() {
+        throw new UnsupportedOperationException();
+    }
+
+    @Override
+    public boolean isShutdown() {
+        return hasBeenShutdown;
+    }
+
+    @Override
+    public boolean isTerminated() {
+        return runningTasks.isEmpty();
+    }
+
+    @Override
+    public boolean awaitTermination(long timeout, TimeUnit unit) throws InterruptedException {
+        return shutdownFuture.await(timeout, unit);
+    }
+
+    private static native void registerNatives();
+
+    private static native int getProxyUnpark(int[] res);
+}
diff --git a/src/java.base/share/classes/com/alibaba/wisp/engine/WispPoller.java b/src/java.base/share/classes/com/alibaba/wisp/engine/WispPoller.java
new file mode 100644
index 00000000000..6bdeffdd2a5
--- /dev/null
+++ b/src/java.base/share/classes/com/alibaba/wisp/engine/WispPoller.java
@@ -0,0 +1,127 @@
+package com.alibaba.wisp.engine;
+
+import jdk.internal.access.SharedSecrets;
+
+import sun.nio.ch.EpollAccess;
+import sun.nio.ch.Net;
+import sun.nio.ch.SelChImpl;
+
+import java.io.IOException;
+import java.nio.channels.SelectableChannel;
+import java.nio.channels.SelectionKey;
+import java.util.concurrent.ConcurrentHashMap;
+
+
+public enum WispPoller {
+
+    INSTANCE;
+
+    private static final int LOW_FD_BOUND = 1024 * 10;
+    private static final int MAX_EVENTS_TO_POLL = 512;
+
+    private final EpollAccess EA;
+    private final int epfd;
+    private Thread thread;
+
+    WispPoller() {
+        if (WispConfiguration.GLOBAL_POLLER) {
+            sun.nio.ch.IOUtil.load();
+            EA = SharedSecrets.getEpollAccess();
+            try {
+                epfd = EA.epollCreate();
+            } catch (IOException e) {
+                throw new ExceptionInInitializerError(e);
+            }
+            thread = new Thread(WispEngine.daemonThreadGroup, this::eventLoop, "Wisp-Poller");
+            thread.setDaemon(true);
+            thread.start();
+        } else {
+            EA = null;
+            epfd = 0;
+        }
+    }
+
+    private WispTask[] fd2TaskLow = new WispTask[LOW_FD_BOUND];
+    private ConcurrentHashMap fd2TaskHigh = new ConcurrentHashMap<>();
+
+    private boolean sanityCheck(int fd, WispTask oldTask, WispTask newTask) {
+        // If timeout happened, when oldTask finished,
+        // the oldTask.ch would be nullified(we didn't get chance to remove it)
+        return (oldTask == null || oldTask == newTask || oldTask.ch == null
+                || ((SelChImpl) oldTask.ch).getFDVal() != fd);
+    }
+
+    private void recordTaskByFD(int fd, WispTask task) {
+        if (fd < LOW_FD_BOUND) {
+            assert sanityCheck(fd, fd2TaskLow[fd], task);
+            fd2TaskLow[fd] = task;
+        } else {
+            assert sanityCheck(fd, fd2TaskHigh.get(fd), task);
+            fd2TaskHigh.put(fd, task);
+        }
+    }
+
+    private WispTask removeTaskByFD(int fd) {
+        WispTask task;
+        if (fd < LOW_FD_BOUND) {
+            task = fd2TaskLow[fd];
+            fd2TaskLow[fd] = null;
+        } else {
+            task = fd2TaskHigh.remove(fd);
+        }
+        return task;
+    }
+
+    void registerEvent(WispTask task, SelectableChannel ch, int event) throws IOException {
+        if (ch == null || !ch.isOpen()) {
+            return;
+        }
+        registerEvent(task, ((SelChImpl) ch).getFDVal(), event);
+    }
+
+    void registerEvent(WispTask task, int fd, int event) throws IOException {
+        int ev = 0;
+        // Translates an interest operation set into a native poll event set
+        if ((event & SelectionKey.OP_READ) != 0) ev |= Net.POLLIN;
+        if ((event & SelectionKey.OP_WRITE) != 0) ev |= Net.POLLOUT;
+        if ((event & SelectionKey.OP_CONNECT) != 0) ev |= Net.POLLCONN;
+        if ((event & SelectionKey.OP_ACCEPT) != 0) ev |= Net.POLLIN;
+        // When the socket is closed, the poll event will be triggered
+        ev |= Net.POLLHUP;
+        // specify the EPOLLONESHOT flag, to tell epoll to disable the associated
+        // file descriptor after the receipt of an event with epoll_wait
+        ev |= EpollAccess.EPOLLONESHOT;
+
+        recordTaskByFD(fd, task);
+        task.setRegisterEventTime();
+        // we can do it multi-thread, because epoll is protected by spin lock in kernel
+        // When the EPOLLONESHOT flag is specified, it is the caller's responsibility to 
+        // rearm the file descriptor using epoll_ctl with EPOLL_CTL_MOD
+        int res = EA.epollCtl(epfd, EpollAccess.EPOLL_CTL_MOD, fd, ev); // rearm
+        if (res != 0 && !(res == EpollAccess.ENOENT && (res = EA.epollCtl(epfd, EpollAccess.EPOLL_CTL_ADD, fd, ev)) == 0)) {
+            removeTaskByFD(fd);
+            task.resetRegisterEventTime();
+            throw new IOException("epoll_ctl " + res);
+        }
+    }
+
+    private void eventLoop() {
+        final long epollArray = EA.allocatePollArray(MAX_EVENTS_TO_POLL);
+        while (true) {
+            try {
+                int n = EA.epollWait(epfd, epollArray, MAX_EVENTS_TO_POLL);
+                while (n-- > 0) {
+                    long eventAddress = EA.getEvent(epollArray, n);
+                    int fd = EA.getDescriptor(eventAddress);
+                    WispTask task = removeTaskByFD(fd);
+                    if (task != null) {
+                        task.countWaitSocketIOTime();
+                        task.jdkUnpark();
+                    }
+                }
+            } catch (Throwable t) {
+                t.printStackTrace();
+            }
+        }
+    }
+}
diff --git a/src/java.base/share/classes/com/alibaba/wisp/engine/WispSysmon.java b/src/java.base/share/classes/com/alibaba/wisp/engine/WispSysmon.java
new file mode 100644
index 00000000000..ca1430b9445
--- /dev/null
+++ b/src/java.base/share/classes/com/alibaba/wisp/engine/WispSysmon.java
@@ -0,0 +1,88 @@
+package com.alibaba.wisp.engine;
+
+
+import jdk.internal.access.JavaLangAccess;
+import jdk.internal.access.SharedSecrets;
+import jdk.internal.access.UnsafeAccess;
+
+import java.util.Comparator;
+import java.util.Set;
+import java.util.concurrent.ConcurrentSkipListSet;
+import java.util.concurrent.TimeUnit;
+
+enum WispSysmon {
+    INSTANCE;
+
+    static {
+        registerNatives();
+    }
+
+    private static Set engines = new ConcurrentSkipListSet<>(new Comparator() {
+        @Override
+        public int compare(WispEngine o1, WispEngine o2) {
+            return o1.threadTask.compareTo(o2.threadTask);
+        }
+    });
+
+    void startDaemon() {
+        if (WispConfiguration.ENABLE_HANDOFF) {
+            Thread thread = new Thread(WispEngine.daemonThreadGroup,
+                    WispSysmon::sysmonLoop, "Wisp-Sysmon");
+            thread.setDaemon(true);
+            thread.start();
+        }
+    }
+
+    void register(WispEngine engine) {
+        if (WispConfiguration.ENABLE_HANDOFF) {
+            engines.add(engine);
+        }
+    }
+
+    private static void sysmonLoop() {
+        final long interval = TimeUnit.MICROSECONDS.toNanos(WispConfiguration.SYSMON_TICK_US);
+        long nextTick = System.nanoTime() + interval;
+        while (true) {
+            final long timeout = nextTick - System.nanoTime();
+            if (timeout > 0) {
+                UA.park0(false, timeout);
+                handleLongOccupation();
+            }
+            nextTick += interval;
+        }
+    }
+
+    /**
+     * Handle a WispTask occupied a carrier thread for long time.
+     *
+     * the strategy is depends on carrier thread status:
+     *
+     * running java code: insert a yield() for next method return
+     * running native code : handOff the carrier (Only supported in wisp2)
+     *
+     */
+    private static void handleLongOccupation() {
+        for (WispEngine engine : engines) {
+            if (engine.terminated) {
+                // remove in iteration is OK for ConcurrentSkipListSet
+                engines.remove(engine);
+            }
+            if (engine.isRunning() && engine.schedTick == engine.lastSchedTick) {
+                if (JLA.isInNative(engine.thread)) {
+                    engine.handOff();
+                } else {
+                    markPreempt(engine.thread);
+                }
+            }
+
+            engine.lastSchedTick = engine.schedTick;
+        }
+    }
+
+    private static native void registerNatives();
+
+    private static native void markPreempt(Thread thread);
+
+    private static final UnsafeAccess UA = SharedSecrets.getUnsafeAccess();
+    private static final JavaLangAccess JLA = SharedSecrets.getJavaLangAccess();
+}
diff --git a/src/java.base/share/classes/com/alibaba/wisp/engine/WispTask.java b/src/java.base/share/classes/com/alibaba/wisp/engine/WispTask.java
new file mode 100644
index 00000000000..4d29a851ce4
--- /dev/null
+++ b/src/java.base/share/classes/com/alibaba/wisp/engine/WispTask.java
@@ -0,0 +1,584 @@
+package com.alibaba.wisp.engine;
+
+import jdk.internal.access.SharedSecrets;
+import jdk.internal.access.UnsafeAccess;
+
+import java.dyn.Coroutine;
+import java.dyn.CoroutineExitException;
+import java.nio.channels.SelectableChannel;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicIntegerFieldUpdater;
+import java.util.function.Supplier;
+
+/**
+ * {@link WispTask} provides high-level semantics of {link @Coroutine}
+ * 

+ * Create {@link WispTask} via {@link WispEngine#dispatch(Runnable)} (Callable, String)} to make
+ * blocking IO operation in {@link WispTask}s to become concurrent.
+ * 

+ * The creator and a newly created {@link WispTask} will automatically have parent-children relationship.
+ * When the child gets blocked on something, the {@link WispEngine} will try to execute parent first.
+ * 

+ * A {@link WispTask}'s exit will wake up the waiting parent.
+ */
+public class WispTask implements Comparable {
+    private final static AtomicInteger idGenerator = new AtomicInteger();
+
+    private static final Map id2Task = new ConcurrentHashMap<>(360);
+    // global table used for all WispEngines
+
+    static WispTask fromId(int id) {
+        WispEngine engine = WispEngine.current();
+        boolean isInCritical0 = engine.isInCritical;
+        engine.isInCritical = true;
+        try {
+            return id2Task.get(id);
+        } finally {
+            engine.isInCritical = isInCritical0;
+        }
+    }
+
+    static void cleanExitedTasks(List tasks) {
+        if (!tasks.isEmpty()) {
+            WispEngine engine = tasks.get(0).engine;
+            boolean isInCritical0 = engine.isInCritical;
+            engine.isInCritical = true;
+            try {
+                tasks.forEach(t -> id2Task.remove(t.id));
+            } finally {
+                engine.isInCritical = isInCritical0;
+            }
+        }
+    }
+
+    static void cleanExitedTask(WispTask task) {
+        WispEngine engine = WispEngine.current();
+        boolean isInCritical0 = engine.isInCritical;
+        engine.isInCritical = true;
+        try {
+            id2Task.remove(task.id);
+        } finally {
+            engine.isInCritical = isInCritical0;
+        }
+    }
+
+    static void trackTask(WispTask task) {
+        WispEngine engine = WispEngine.current();
+        boolean isInCritical0 = engine.isInCritical;
+        engine.isInCritical = true;
+        try {
+            id2Task.put(task.id, task);
+        } finally {
+            engine.isInCritical = isInCritical0;
+        }
+    }
+
+
+    private final int id;
+
+    enum Status {
+        RUNNABLE,   // could be run
+        BLOCKED,    // waiting for IO or timer
+        ZOMBIE      // exited
+    }
+
+
+    private Runnable runnable; // runnable for created task
+    Supplier command; // this task is a pseudo task used to pass command
+
+    /**
+     * Task is running in that engine.
+     */
+    WispEngine engine;
+
+    private String name;
+    final Coroutine ctx;                // the low-level coroutine implement
+    Status status = Status.RUNNABLE;
+    SelectableChannel ch;               // the interesting channel
+    TimeOut timeOut;                    // related timer
+    ClassLoader ctxClassLoader;
+
+    WispTask parent;
+    private boolean isThreadTask;
+    private boolean isThreadAsWisp;
+
+    private Thread threadWrapper;       // thread returned by Thread::currentThread()
+    private volatile int interrupted;   // 0 means not interrupted
+    private volatile int alreadyCheckNativeInterrupt;
+
+    private volatile int jdkParkStatus;
+    private volatile int jvmParkStatus;
+    volatile int stealLock;
+    private WispTask from;
+    final StealAwareRunnable resumeEntry;
+    // counter printed by jstack
+    private int activeCount;
+    int stealCount;
+    int stealFailureCount;
+    long enqueueTime;
+    long parkTime;
+    long blockingTime;
+    long registerEventTime;
+    /**
+     * reduce the cost of deciding weather task is in queue.
+     */
+    final AtomicBoolean enqueued = new AtomicBoolean(false);
+
+    WispTask(WispEngine engine, Coroutine ctx, boolean isRealTask, boolean isThreadTask) {
+        this.isThreadTask = isThreadTask;
+        this.id     = isRealTask ? idGenerator.addAndGet(1) : -1;
+        this.engine = engine;
+        if (isRealTask) {
+            this.ctx = ctx != null ? ctx :
+                    new Coroutine(WispConfiguration.STACK_SIZE) {
+                        @Override
+                        protected void run() {
+                            coroutineCacheLoop();
+                        }
+                    };
+            this.ctx.setWispTask(id, this, engine);
+        } else {
+            this.ctx = null;
+        }
+        resumeEntry = isThreadTask ? null : engine.createResumeEntry(this);
+    }
+
+    void reset(Runnable runnable, WispTask parent, String name, Thread thread, ClassLoader ctxLoader) {
+        assert ctx != null;
+        this.status = Status.RUNNABLE;
+        this.runnable = runnable;
+        this.parent = parent;
+        this.name   = name;
+        this.interrupted = 0;
+        this.ctxClassLoader = ctxLoader;
+        ch          = null;
+        enqueueTime = 0;
+        parkTime = 0;
+        blockingTime = 0;
+        registerEventTime = 0;
+        enqueued.lazySet(false);
+
+        activeCount = 0;
+        stealCount  = 0;
+        stealFailureCount = 0;
+
+        // thread status
+        if (thread != null) { // calling from Thread.start()
+            NATIVE_INTERRUPTED_UPDATER.lazySet(this, 1);
+            isThreadAsWisp = true;
+            WispEngine.JLA.setWispTask(thread, this);
+            threadWrapper = thread;
+        } else {
+            // for WispThreadWrapper, skip native interrupt check
+            NATIVE_INTERRUPTED_UPDATER.lazySet(this, 0);
+            isThreadAsWisp = false;
+            if (threadWrapper == null) {
+                threadWrapper = new WispThreadWrapper(this);
+            }
+            WispEngine.JLA.setWispAlive(threadWrapper, true);
+        }
+        assert WispEngine.JLA.getWispTask(threadWrapper) == this;
+
+        if (!isThreadTask() && name != null && !threadWrapper.getName().equals(name)) {
+            threadWrapper.setName(name);
+        }
+    }
+
+    private void coroutineCacheLoop() {
+        while (true) {
+            assert !WispConfiguration.WISP_USE_STEAL_LOCK || stealLock != 0;
+            assert WispEngine.current() == engine;
+            assert engine.current == this;
+            if (runnable != null) {
+                Throwable throwable = null;
+                try {
+                    runnable.run();
+                } catch (Throwable t) {
+                    throwable = t;
+                } finally {
+                    assert timeOut == null;
+                    runnable = null;
+                    WispEngine.JLA.setWispAlive(threadWrapper, false);
+                    if (isThreadAsWisp) {
+                        WispEngine.JLA.threadExit(threadWrapper);
+                        synchronized (threadWrapper) {
+                            threadWrapper.notifyAll();
+                        }
+                        threadWrapper = null; // else WispThreadWrapper could be reused
+                    }
+                    if (throwable instanceof CoroutineExitException) {
+                        throw (CoroutineExitException) throwable;
+                    }
+                    engine.taskExit();
+                }
+            } else {
+                engine.yieldOnBlocking();
+            }
+        }
+    }
+
+    /**
+     * Switch task. we need the information of {@code from} task param
+     * to do classloader switch etc..
+     *
+     * {@link #stealLock} is used in {@link Wisp2Engine#steal(WispTask, WispEngine, boolean)} .
+     */
+    static void switchTo(WispTask current, WispTask next) {
+        assert next.ctx != null;
+        assert current.engine == next.engine;
+        next.activeCount++;
+        if (WispConfiguration.WISP_USE_STEAL_LOCK) {
+            next.from = current;
+            STEAL_LOCK_UPDATER.lazySet(next, 1);
+            // store load barrier is not necessary
+        }
+        Coroutine.yieldTo(next.ctx);
+        if (WispConfiguration.WISP_USE_STEAL_LOCK) {
+            assert current.stealLock != 0;
+            STEAL_LOCK_UPDATER.lazySet(current.from, 0);
+        }
+        assert WispEngine.current() == current.engine;
+        assert current.engine.current == current;
+    }
+
+    /**
+     * @return {@code false} if current {@link WispTask} is thread-emulated.
+     */
+    boolean isThreadTask() {
+        return isThreadTask;
+    }
+
+    TimeOut getTimeOut() {
+        return timeOut;
+    }
+
+    /**
+     * Let currently executing task sleep for specified number of milliseconds.
+     * 

+     * May be wakened up early by an available IO.
+     */
+    static void sleep(long ms) {
+        if (ms < 0) throw new IllegalArgumentException();
+
+        if (ms == 0) {
+            WispEngine.current().yield();
+        } else {
+            WispEngine.current().unregisterEvent();
+            jdkPark(TimeUnit.MILLISECONDS.toNanos(ms));
+        }
+    }
+
+    @Override
+    public String toString() {
+        return "WispTask" + id + "(" +
+                "name=" + name + ')' +
+                "{status=" + status + "/" +
+                jdkParkStatus + ", " +
+                "enqueued=" + enqueued +
+                '}';
+    }
+
+    public String getName() {
+        return name;
+    }
+
+
+    private static final int
+            WAITING = -1,   // was blocked
+            FREE = 0,       // the Initial Park status
+            PERMITTED = 1;  // another task give a permit to make the task not block at next park()
+
+
+    boolean isRunnable() {
+        return status == Status.RUNNABLE;
+    }
+
+    boolean isAlive() {
+        return status != Status.ZOMBIE;
+    }
+
+    /**
+     * If a permit is available, it will be consumed and this function returns
+     * immediately; otherwise
+     * current task will become blocked until {@link #unpark()} ()} happens.
+     *
+     * @param timeoutNano <= 0 park forever
+     *                    else park with given timeout
+     */
+    private void parkInternal(long timeoutNano, boolean fromJvm) {
+        final AtomicIntegerFieldUpdater statusUpdater = fromJvm ? JVM_PARK_UPDATER : JDK_PARK_UPDATER;
+
+        final boolean isInCritical0 = engine.isInCritical;
+        engine.isInCritical = true;
+        try {
+            engine.getCounter().incrementParkCount();
+            for (;;) {
+                int s = statusUpdater.get(this);
+                assert s != WAITING; // if parkStatus == WAITING, should already blocked
+
+                if (s == FREE && statusUpdater.compareAndSet(this, FREE, WAITING)) {
+                    // may become PERMITTED here; need retry.
+                    // another thread unpark here is ok:
+                    // current task is put to unpark queue,
+                    // and will wake up eventually
+                    if (WispEngine.runningAsCoroutine(threadWrapper) && timeoutNano > 0) {
+                        engine.addTimer(timeoutNano + System.nanoTime(), fromJvm);
+                    }
+                    engine.isInCritical = isInCritical0;
+                    try {
+                        if (WispEngine.runningAsCoroutine(threadWrapper)) {
+                            setParkTime();
+                            engine.yieldOnBlocking();
+                        } else {
+                            UA.park0(false, timeoutNano < 0 ? 0 : timeoutNano);
+                        }
+                    } finally {
+                        engine.isInCritical = true;
+                        if (timeoutNano > 0) {
+                            engine.cancelTimer();
+                        }
+                        // we'may direct wakeup by current engine
+                        // the statue may be still WAITING..
+                        statusUpdater.lazySet(this, FREE);
+                    }
+                    break;
+                } else if (s == PERMITTED &&
+                        (statusUpdater.compareAndSet(this, PERMITTED, FREE))) {
+                    // consume the permit
+                    break;
+                }
+            }
+        } finally {
+            engine.isInCritical = isInCritical0;
+        }
+    }
+
+    /**
+     * If the thread was blocked on {@link #park(long)} then it will unblock.
+     * Otherwise, its next call to {@link #park(long)} is guaranteed not to block.
+     */
+    private void unparkInternal(boolean fromJvm) {
+        AtomicIntegerFieldUpdater statusUpdater = fromJvm ? JVM_PARK_UPDATER : JDK_PARK_UPDATER;
+        for (;;) {
+            int s = statusUpdater.get(this);
+            if (s == WAITING && statusUpdater.compareAndSet(this, WAITING, FREE)) {
+                if (WispEngine.runningAsCoroutine(threadWrapper)) {
+                    recordOnUnpark(fromJvm);
+                    engine.wakeupTask(this);
+                } else {
+                    UA.unpark0(threadWrapper);
+                }
+                break;
+            } else if (s == PERMITTED ||
+                    (s == FREE && statusUpdater.compareAndSet(this, FREE, PERMITTED))) {
+                // add a permit
+                break;
+            }
+        }
+    }
+
+    /**
+     * Park Invoked by jdk, include IO, JUC etc..
+     */
+    static void jdkPark(long timeoutNano) {
+        WispEngine.current().getCurrentTask().parkInternal(timeoutNano, false);
+    }
+
+    void jdkUnpark() {
+        unparkInternal(false);
+    }
+
+    /**
+     * Invoked by VM to support coroutine switch in object monitor case.
+     */
+    private static void park(long timeoutNano) {
+        WispEngine.current().getCurrentTask().parkInternal(timeoutNano, true);
+    }
+
+    void unpark() {
+        unparkInternal(true);
+    }
+
+    // direct called by jvm runtime if UseDirectUnpark
+    static void unparkById(int id) {
+        WispEngine engine = WispEngine.current();
+        boolean isInCritical0 = engine.isInCritical;
+        engine.isInCritical = true;
+        try {
+            WispTask t = id2Task.get(id);
+            if (t != null) {
+                t.unpark();
+            }
+        } finally {
+            engine.isInCritical = isInCritical0;
+        }
+    }
+
+    void interrupt() {
+        // For JSR166. Unpark even if interrupt status was already set.
+        interrupted = 1;
+        unpark();
+        jdkUnpark();
+    }
+
+    private static void interruptById(int id) {
+        WispTask t = fromId(id);
+        if (t != null) {
+            t.interrupt();
+        }
+    }
+
+    boolean isInterrupted() {
+        return interrupted != 0;
+    }
+
+    boolean testInterruptedAndClear(boolean clear) {
+        boolean nativeInterrupt = false;
+        if (alreadyCheckNativeInterrupt == 0 && // only do it once
+                NATIVE_INTERRUPTED_UPDATER.compareAndSet(this, 0, 1) &&
+                !isInterrupted()) {
+            nativeInterrupt = checkAndClearNativeInterruptForWisp(threadWrapper);
+        }
+        boolean res = interrupted != 0 || nativeInterrupt;
+        if (res && clear) {
+            INTERRUPTED_UPDATER.lazySet(this, 0);
+        }
+        // return old interrupt status.
+        return res;
+    }
+
+    void lazySetInterrupted(boolean interrupted) {
+        INTERRUPTED_UPDATER.lazySet(this, interrupted ? 1 : 0);
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        WispTask t = (WispTask) o;
+        return Objects.equals(id, t.id);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(id);
+    }
+
+    public Thread getThreadWrapper() {
+        return threadWrapper;
+    }
+
+    void setThreadWrapper(Thread thread) {
+        threadWrapper = thread;
+        WispEngine.JLA.setWispTask(thread, this);
+    }
+
+    @Override
+    public int compareTo(WispTask o) {
+        return Integer.compare(this.id, o.id);
+    }
+
+    void updateEnqueueTime() {
+        if (!WispConfiguration.WISP_PROFILE_DETAIL) {
+            return;
+        }
+        // For pseudo task, use another flow to record the enque time
+        if (command != null) {
+            return;
+        }
+        // in wisp2, if the task is stealed unsuccessfully, it will be put into queue again
+        if (enqueueTime != 0) {
+            assert WispConfiguration.WISP_VERSION == 2;
+            return;
+        }
+        enqueueTime = System.nanoTime();
+    }
+
+    long getEnqueueTime() {
+        return enqueueTime;
+    }
+
+    void resetEnqueueTime() {
+        enqueueTime = 0;
+    }
+
+    void setRegisterEventTime() {
+        // only count the time which is spent on WispTask by service
+        registerEventTime = (!WispConfiguration.WISP_PROFILE_DETAIL || isThreadTask) ? 0 : System.nanoTime();
+    }
+
+    void resetRegisterEventTime() {
+        registerEventTime = 0;
+    }
+
+    void countWaitSocketIOTime() {
+        if (registerEventTime != 0) {
+            engine.counter.incrementTotalWaitSocketIOTime(System.nanoTime() - registerEventTime);
+            registerEventTime = 0;
+        }
+    }
+
+    void setParkTime() {
+        parkTime = (!WispConfiguration.WISP_PROFILE_DETAIL || isThreadTask) ? 0 : System.nanoTime();
+    }
+
+    /* When unpark is called, the time is set.
+     * Since the unpark may be called by non-carrier thread, the count is delayed.
+     */
+    private void recordOnUnpark(boolean fromJVM) {
+        if (!WispConfiguration.WISP_PROFILE_DETAIL) {
+            return;
+        }
+        if (parkTime != 0) {
+            blockingTime = System.nanoTime() - parkTime;
+            if (blockingTime < 0) {
+                blockingTime = 0;
+            }
+            parkTime = 0;
+        }
+        if (fromJVM) {
+            engine.counter.incrementUnparkFromJvmCount();
+        }
+    }
+
+    void countExecutionTime(long beginTime) {
+        // TaskExit set beginTime to 0, and calls yieldOnBlocking,
+        // then beginTime is 0. It need to skip it.
+        if (!WispConfiguration.WISP_PROFILE_DETAIL || beginTime == 0) {
+            return;
+        }
+        engine.counter.incrementTotalExecutionTime(System.nanoTime() - beginTime);
+        if (blockingTime != 0) {
+            engine.counter.incrementTotalBlockingTime(blockingTime);
+            blockingTime = 0;
+        }
+    }
+
+
+    private static final AtomicIntegerFieldUpdater JVM_PARK_UPDATER;
+    private static final AtomicIntegerFieldUpdater JDK_PARK_UPDATER;
+    private static final AtomicIntegerFieldUpdater INTERRUPTED_UPDATER;
+    private static final AtomicIntegerFieldUpdater NATIVE_INTERRUPTED_UPDATER;
+    private static final AtomicIntegerFieldUpdater STEAL_LOCK_UPDATER;
+    private static final UnsafeAccess UA = SharedSecrets.getUnsafeAccess();
+
+    private static native void registerNatives();
+
+    // only for wisp to clear the native interrupt, for parallel interrupt problem.
+    private static native boolean checkAndClearNativeInterruptForWisp(Thread cur);
+
+    static {
+        JVM_PARK_UPDATER = AtomicIntegerFieldUpdater.newUpdater(WispTask.class, "jvmParkStatus");
+        JDK_PARK_UPDATER = AtomicIntegerFieldUpdater.newUpdater(WispTask.class, "jdkParkStatus");
+        INTERRUPTED_UPDATER = AtomicIntegerFieldUpdater.newUpdater(WispTask.class, "interrupted");
+        NATIVE_INTERRUPTED_UPDATER = AtomicIntegerFieldUpdater.newUpdater(WispTask.class, "alreadyCheckNativeInterrupt");
+        STEAL_LOCK_UPDATER = AtomicIntegerFieldUpdater.newUpdater(WispTask.class, "stealLock");
+        registerNatives();
+    }
+}
diff --git a/src/java.base/share/classes/com/alibaba/wisp/engine/WispThreadWrapper.java b/src/java.base/share/classes/com/alibaba/wisp/engine/WispThreadWrapper.java
new file mode 100644
index 00000000000..8da341673c4
--- /dev/null
+++ b/src/java.base/share/classes/com/alibaba/wisp/engine/WispThreadWrapper.java
@@ -0,0 +1,82 @@
+package com.alibaba.wisp.engine;
+
+
+import jdk.internal.access.JavaLangAccess;
+import jdk.internal.access.SharedSecrets;
+import jdk.internal.reflect.CallerSensitive;
+
+import java.dyn.CoroutineSupport;
+
+/**
+ * An wrapper of {@link Thread} let every {@link WispTask} get different thread
+ * object from {@link Thread#currentThread()}.
+ * In this way, we make the listed class (not only but include) behave expected without
+ * changing their code.
+ * 

+ * 1. {@link ThreadLocal}
+ * 2. {@link java.util.concurrent.locks.AbstractQueuedSynchronizer} based synchronizer
+ * 3. Netty's judgment of weather we are running in it's worker thread.
+ */
+class WispThreadWrapper extends Thread {
+
+    WispThreadWrapper(WispTask task) {
+        JLA.setWispTask(this, task);
+        setName(task.getName());
+    }
+
+    @Override
+    public CoroutineSupport getCoroutineSupport() {
+        return JLA.getWispTask(this).engine.thread.getCoroutineSupport();
+    }
+
+    @Override
+    public void start() {
+        throw new UnsupportedOperationException();
+    }
+
+    @Override
+    @CallerSensitive
+    public ClassLoader getContextClassLoader() {
+        return JLA.getWispTask(this).ctxClassLoader;
+    }
+
+    @Override
+    public void setContextClassLoader(ClassLoader cl) {
+        JLA.getWispTask(this).ctxClassLoader = cl;
+    }
+
+    @Override
+    public StackTraceElement[] getStackTrace() {
+        return super.getStackTrace();
+    }
+
+    @Override
+    public long getId() {
+        return super.getId();
+    }
+
+    @Override
+    public State getState() {
+        return JLA.getWispTask(this).engine.thread.getState();
+    }
+
+    @Override
+    public UncaughtExceptionHandler getUncaughtExceptionHandler() {
+        return JLA.getWispTask(this).engine.thread.getUncaughtExceptionHandler();
+    }
+
+    @Override
+    public void setUncaughtExceptionHandler(UncaughtExceptionHandler eh) {
+        JLA.getWispTask(this).engine.thread.setUncaughtExceptionHandler(eh);
+    }
+
+    @Override
+    public String toString() {
+        return "WispThreadWrapper{" +
+                "wispTask=" + JLA.getWispTask(this) +
+                '}';
+    }
+
+    private static final JavaLangAccess JLA = SharedSecrets.getJavaLangAccess();
+
+}
diff --git a/src/java.base/share/classes/com/alibaba/wisp/engine/WispWorkerContainer.java b/src/java.base/share/classes/com/alibaba/wisp/engine/WispWorkerContainer.java
new file mode 100644
index 00000000000..b5e1a43cbc1
--- /dev/null
+++ b/src/java.base/share/classes/com/alibaba/wisp/engine/WispWorkerContainer.java
@@ -0,0 +1,113 @@
+package com.alibaba.wisp.engine;
+
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.Future;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
+
+/**
+ * Lots of networking library maintain their own thread model.
+ * Instead, we create CPU-number befitted threads and letting user's
+ * code running in a coroutine of our maintained threads.
+ */
+public enum WispWorkerContainer {
+    INSTANCE;
+
+    public static final String WISP_THREAD_NAME_PREFIX = "WISP_WORKER_";
+
+    private List workers = new ArrayList<>();
+    private ConcurrentHashMap id2dispatcher = new ConcurrentHashMap<>();
+
+    public List getWorkers() {
+        return workers;
+    }
+
+    WispWorkerContainer() {
+        if (WispConfiguration.WISP_VERSION == 2) {
+            return;
+        }
+        // pre-start threads.
+        CountDownLatch countDownLatch = new CountDownLatch(WispConfiguration.WORKER_COUNT);
+        List workerList = IntStream.range(0, WispConfiguration.WORKER_COUNT).boxed().map(i -> {
+            Thread t = new Thread(null, new Runnable() { // convert to lambda leading to block(jvm BUG?)
+                @Override
+                public void run() {
+                    WispEngine engine = WispEngine.current(); // create engine
+                    countDownLatch.countDown();
+                    engine.yieldOnBlocking();
+                    // wait task submit
+                }
+            }, WISP_THREAD_NAME_PREFIX + i);
+            t.start();
+            return t;
+        }).collect(Collectors.toList());
+        try { // wait engine create finish
+            countDownLatch.await();
+        } catch (InterruptedException e) {
+            e.printStackTrace();
+        }
+        workers = workerList.stream().map(WispEngine.JLA::getWispEngine).collect(Collectors.toList());
+    }
+
+    /**
+     * Create a coroutine running in the global workers
+     *
+     * @param group in which coroutines will be dispatched in Round-robin into workers.
+     * @param name  name as return value for Thread.currentThread().getName()
+     * @param r     target code
+     * @param t     as return value for Thread.currentThread()
+     */
+    public Future dispatch(String group, String name, Runnable r, Thread t) {
+        WispEngine wispEngine;
+        if (WispConfiguration.WISP_VERSION == 2) {
+            wispEngine = WispEngine.current();
+        } else {
+            Dispatcher dispatcher = id2dispatcher.get(group);
+            if (dispatcher == null) {
+                id2dispatcher.putIfAbsent(group, new Dispatcher());
+                dispatcher = id2dispatcher.get(group);
+            }
+            wispEngine = dispatcher.nextEngine();
+        }
+        return wispEngine.summitTask(r, name, t);
+    }
+
+    private class Dispatcher {
+        AtomicInteger cnt = new AtomicInteger(0);
+        int[] order;
+
+        Dispatcher() {
+            generateOrder();
+        }
+
+        private void generateOrder() {
+            List list = IntStream.range(0, workers.size()).boxed().collect(Collectors.toList());
+            Collections.shuffle(list);
+            order = list.stream().mapToInt(i -> i).toArray();
+        }
+
+        WispEngine nextEngine() {
+            return workers.get(order[cnt.getAndIncrement() % order.length]);
+        }
+    }
+
+    /**
+     * make group for thread according to its name.
+     *
+     * Example:
+     * Druid-ConnectionPool-CreateScheduler--1-Thread-4758
+     * -->
+     * DruidConnectionPoolCreateSchedulerThread
+     */
+    static String getThreadUsage(String threadName) {
+        return Arrays.stream(threadName.split("[# \\-_.]")).filter(s -> !s.matches("\\d+(\\.\\d+)?"))
+                .reduce((s1, s2) -> s1 + s2).orElse("null");
+    }
+}
diff --git a/src/java.base/share/classes/com/alibaba/wisp/util/io/SleepSelector.java b/src/java.base/share/classes/com/alibaba/wisp/util/io/SleepSelector.java
new file mode 100644
index 00000000000..c91ce53823e
--- /dev/null
+++ b/src/java.base/share/classes/com/alibaba/wisp/util/io/SleepSelector.java
@@ -0,0 +1,158 @@
+package com.alibaba.wisp.util.io;
+
+
+import java.io.IOException;
+import java.lang.reflect.Field;
+import java.lang.reflect.Method;
+import java.nio.channels.SelectionKey;
+import java.nio.channels.Selector;
+import java.nio.channels.spi.AbstractSelectableChannel;
+import java.nio.channels.spi.AbstractSelector;
+import java.util.AbstractSet;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.Set;
+
+/**
+ * For the IO intensive workloads, we need to wakeup IO threads frequently via
+ * {@link Selector#wakeup()}, which is an expensive operation(lock, IO).
+ * 

+ * So we use a "sleep" strategy to solve this problem:
+ * If IO thread is busy, the call on  {@link Selector#wakeup()} is ignored as
+ * IO thread will be definitely wakened  by IO operations self, which is already registered in {@link Selector}.
+ */
+@SuppressWarnings("removal")
+public class SleepSelector extends AbstractSelector {
+
+    private static final Field cancelledKeysField;
+    private static final Method registerMethod;
+    private static int ENTER_SLEEP_CNT;
+    private static int EXIT_SLEEP_CNT;
+    private static int SAFE_INTERVAL;
+
+    static {
+        try {
+            cancelledKeysField = AbstractSelector.class.getDeclaredField("cancelledKeys");
+            cancelledKeysField.setAccessible(true);
+            registerMethod = AbstractSelector.class.getDeclaredMethod("register",
+                    AbstractSelectableChannel.class, int.class, Object.class);
+            registerMethod.setAccessible(true);
+        } catch (ReflectiveOperationException e) {
+            throw new Error(e);
+        }
+        java.security.AccessController.doPrivileged(
+                new java.security.PrivilegedAction() {
+                    @Override
+                    public Void run() {
+                        ENTER_SLEEP_CNT = Integer.getInteger("com.alibaba.selector.sleep.onCnt", 18);
+                        EXIT_SLEEP_CNT = Integer.getInteger("com.alibaba.selector.sleep.offCnt", 12);
+                        SAFE_INTERVAL = Integer.getInteger("com.alibaba.selector.safe.interval", 120);
+                        return null;
+                    }
+                });
+    }
+
+    // sleep selector algorithm control variables
+    private volatile boolean needKick = true; // should we wakeup selector?
+    private int sleepInterval = 1;
+    private int continuousEvent = 0;
+    private int continuousNonEvent = 0;
+
+    private final AbstractSelector target;
+
+    @SuppressWarnings("unchecked")
+    public SleepSelector(AbstractSelector target) {
+        super(target.provider());
+        this.target = target;
+        try {
+            Set targetCk = (Set) cancelledKeysField.get(target);
+            cancelledKeysField.set(this, new AbstractSet() {
+                @Override
+                public boolean add(SelectionKey sk) {
+                    synchronized (targetCk) {
+                        return targetCk.add(sk);
+                    }
+                }
+
+                @Override
+                public Iterator iterator() {
+                    return Collections.emptyIterator();
+                }
+
+                @Override
+                public int size() {
+                    return 0;
+                }
+            });
+        } catch (ReflectiveOperationException e) {
+            throw new IllegalStateException(e);
+        }
+    }
+
+    @Override
+    protected void implCloseSelector() throws IOException {
+        target.close();
+    }
+
+    @Override
+    protected SelectionKey register(AbstractSelectableChannel ch, int ops, Object att) {
+        try {
+            return (SelectionKey) registerMethod.invoke(target, ch, ops, att);
+        } catch (ReflectiveOperationException e) {
+            throw new IllegalStateException(e);
+        }
+    }
+
+    @Override
+    public Set keys() {
+        return target.keys();
+    }
+
+    @Override
+    public Set selectedKeys() {
+        return target.selectedKeys();
+    }
+
+    @Override
+    public int selectNow() throws IOException {
+        return target.selectNow();
+    }
+
+    @Override
+    public int select(long timeout) throws IOException {
+        return updateKickStatus(target.select(Math.min(timeout, sleepInterval)));
+    }
+
+    @Override
+    public int select() throws IOException {
+        return updateKickStatus(target.select(sleepInterval));
+    }
+
+    private int updateKickStatus(final int nKeys) {
+        if (nKeys > 0) {
+            sleepInterval = 1;
+            continuousNonEvent = 0;
+            if (++continuousEvent >= ENTER_SLEEP_CNT && needKick) {
+                needKick = false;
+            }
+        } else {
+            continuousEvent = 0;
+            if (++continuousNonEvent >= EXIT_SLEEP_CNT && !needKick) {
+                needKick = true;
+            }
+            if (continuousNonEvent >= EXIT_SLEEP_CNT + 2
+                    // +2 is enough to ensure needKick = true is already seen by other threads.
+                    && sleepInterval == 1) {
+                // we're really not busy, use longer sleep interval to save CPU
+                sleepInterval = SAFE_INTERVAL;
+            }
+        }
+        return nKeys;
+    }
+
+    @Override
+    public Selector wakeup() {
+        if (needKick) target.wakeup();
+        return this;
+    }
+}
diff --git a/src/java.base/share/classes/com/alibaba/wisp/util/io/WispInputStream.java b/src/java.base/share/classes/com/alibaba/wisp/util/io/WispInputStream.java
new file mode 100644
index 00000000000..a178f07e5bb
--- /dev/null
+++ b/src/java.base/share/classes/com/alibaba/wisp/util/io/WispInputStream.java
@@ -0,0 +1,128 @@
+package com.alibaba.wisp.util.io;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.Socket;
+import java.net.SocketTimeoutException;
+import java.nio.ByteBuffer;
+import java.nio.channels.SelectionKey;
+import java.nio.channels.SocketChannel;
+import java.util.concurrent.TimeUnit;
+
+import jdk.internal.access.SharedSecrets;
+import jdk.internal.access.WispEngineAccess;
+
+// Make a nio channel implemented Socket's InputStream be like a plain socket's and yield on block
+
+public class WispInputStream extends InputStream {
+
+    private static WispEngineAccess WEA = SharedSecrets.getWispEngineAccess();
+
+    protected final SocketChannel ch;
+    protected Socket so;
+    private ByteBuffer bb = null;
+    private byte[] bs = null;           // Invoker's previous array
+    private byte[] b1 = null;
+
+    private ByteBuffer readAhead = null;
+
+    public WispInputStream(SocketChannel ch, Socket so) {
+        assert !ch.isBlocking();
+        this.ch = ch;
+        this.so = so;
+    }
+
+    @Override
+    public int read() throws IOException {
+        if (b1 == null)
+            b1 = new byte[1];
+        int n = this.read(b1);
+        if (n == 1)
+            return b1[0] & 0xff;
+        return -1;
+    }
+
+    @Override
+    public int read(byte[] bs, int off, int len)
+            throws IOException {
+        if (len <= 0 || off < 0 || off + len > bs.length) {
+            if (len == 0) {
+                return 0;
+            }
+            throw new ArrayIndexOutOfBoundsException();
+        }
+
+        ByteBuffer bb = ((this.bs == bs) ? this.bb : ByteBuffer.wrap(bs));
+
+        bb.limit(Math.min(off + len, bb.capacity()));
+        bb.position(off);
+        this.bb = bb;
+        this.bs = bs;
+        return read(bb);
+    }
+
+
+    protected int read(ByteBuffer bb)
+            throws IOException {
+        int n;
+
+        if (readAhead != null && readAhead.hasRemaining()) {
+            if (bb.remaining() >= readAhead.remaining()) {
+                n = readAhead.remaining();
+                bb.put(readAhead);
+                return n;
+            } else {
+                n = bb.remaining();
+                for (int i = 0; i < n; i++) {
+                    bb.put(readAhead.get());
+                }
+                return n;
+            }
+        }
+
+        try {
+            if ((n = ch.read(bb)) != 0) {
+                return n;
+            }
+
+            if (so.getSoTimeout() > 0) {
+                WEA.addTimer(System.nanoTime() + TimeUnit.MILLISECONDS.toNanos(so.getSoTimeout()));
+            }
+
+            do {
+                WEA.registerEvent(ch, SelectionKey.OP_READ);
+                WEA.park(-1);
+
+                if (so.getSoTimeout() > 0 && WEA.isTimeout()) {
+                    throw new SocketTimeoutException("time out");
+
+                }
+            } while ((n = ch.read(bb)) == 0);
+        } finally {
+            if (so.getSoTimeout() > 0) {
+                WEA.cancelTimer();
+            }
+            WEA.unregisterEvent();
+        }
+
+        return n;
+    }
+
+    public int available() throws IOException {
+        if (readAhead == null) {
+            readAhead = ByteBuffer.allocate(4096);
+        } else if (readAhead.hasRemaining()) {
+            return readAhead.remaining();
+        }
+
+        readAhead.clear();
+        ch.read(readAhead);
+        readAhead.flip();
+
+        return readAhead.remaining();
+    }
+
+    public void close() throws IOException {
+        ch.close();
+    }
+}
diff --git a/src/java.base/share/classes/com/alibaba/wisp/util/io/WispOutputStream.java b/src/java.base/share/classes/com/alibaba/wisp/util/io/WispOutputStream.java
new file mode 100644
index 00000000000..402af0ac741
--- /dev/null
+++ b/src/java.base/share/classes/com/alibaba/wisp/util/io/WispOutputStream.java
@@ -0,0 +1,95 @@
+package com.alibaba.wisp.util.io;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.net.Socket;
+import java.net.SocketTimeoutException;
+import java.nio.ByteBuffer;
+import java.nio.channels.SelectionKey;
+import java.nio.channels.SocketChannel;
+import java.util.concurrent.TimeUnit;
+
+import jdk.internal.access.SharedSecrets;
+import jdk.internal.access.WispEngineAccess;
+
+// Make a nio channel implemented Socket's OutputStream be like a plain socket's and yield on block
+
+public class WispOutputStream extends OutputStream {
+
+    private static WispEngineAccess WEA = SharedSecrets.getWispEngineAccess();
+
+    protected final SocketChannel ch;
+    protected final Socket so;
+    private ByteBuffer bb = null;
+    private byte[] bs = null;       // Invoker's previous array
+    private byte[] b1 = null;
+
+
+    public WispOutputStream(SocketChannel ch, Socket so) {
+        assert !ch.isBlocking();
+        this.ch = ch;
+        this.so = so;
+    }
+
+    @Override
+    public void write(int b) throws IOException {
+        if (b1 == null)
+            b1 = new byte[1];
+        b1[0] = (byte) b;
+        this.write(b1);
+    }
+
+    @Override
+    public void write(byte[] bs, int off, int len)
+            throws IOException
+    {
+        if (len <= 0 || off < 0 || off + len > bs.length) {
+            if (len == 0) {
+                return;
+            }
+            throw new ArrayIndexOutOfBoundsException();
+        }
+        ByteBuffer bb = ((this.bs == bs) ? this.bb : ByteBuffer.wrap(bs));
+        bb.limit(Math.min(off + len, bb.capacity()));
+        bb.position(off);
+        this.bb = bb;
+        this.bs = bs;
+
+        write(bb);
+    }
+
+    protected void write(ByteBuffer bb)
+            throws IOException {
+
+        try {
+            int writeLen = bb.remaining();
+            if (ch.write(bb) == writeLen) {
+                return;
+            }
+
+            if (so.getSoTimeout() > 0) {
+                WEA.addTimer(System.nanoTime() + TimeUnit.MILLISECONDS.toNanos(so.getSoTimeout()));
+            }
+
+            do {
+                WEA.registerEvent(ch, SelectionKey.OP_WRITE);
+                WEA.park(-1);
+
+                if (so.getSoTimeout() > 0 && WEA.isTimeout()) {
+                    throw new SocketTimeoutException("time out");
+                }
+                ch.write(bb);
+            } while (bb.remaining() > 0);
+        } finally {
+            if (so.getSoTimeout() > 0) {
+                WEA.cancelTimer();
+            }
+            WEA.unregisterEvent();
+        }
+    }
+
+    public void close() throws IOException {
+        ch.close();
+    }
+
+}
diff --git a/src/java.base/share/classes/java/dyn/Coroutine.java b/src/java.base/share/classes/java/dyn/Coroutine.java
index a018fafb165..7f2e257e8f5 100644
--- a/src/java.base/share/classes/java/dyn/Coroutine.java
+++ b/src/java.base/share/classes/java/dyn/Coroutine.java
@@ -25,6 +25,8 @@
 
 package java.dyn;
 
+import jdk.internal.access.SharedSecrets;
+
 /**
  * Implementation of symmetric coroutines. A Coroutine will take part in thread-wide scheduling of coroutines. It transfers control to
  * the next coroutine whenever yield is called.
@@ -42,8 +44,8 @@
  * 		for (int i = 0; i < 10; i++) {
  * 			System.out.println(i);
  * 			yield();
- * 		}
- * 	}
+ *        }
+ *    }
  * }
  * 

  *
@@ -64,86 +66,117 @@
  * @author Lukas Stadler
  */
 public class Coroutine extends CoroutineBase {
-	private final Runnable target;
+    private final Runnable target;
 
-	Coroutine next;
-	Coroutine last;
+    Coroutine next;
+    Coroutine last;
 
     /**
      * Allocates a new {@code Coroutine} object.
      */
-	public Coroutine() {
-		this.target = null;
-		threadSupport.addCoroutine(this, -1);
-	}
+    public Coroutine() {
+        this.target = null;
+        threadSupport.addCoroutine(this, -1);
+    }
 
     /**
      * Allocates a new {@code Coroutine} object.
+	 * 
      * @param target the runnable
      */
-	public Coroutine(Runnable target) {
-		this.target = target;
-		threadSupport.addCoroutine(this, -1);
-	}
+    public Coroutine(Runnable target) {
+        this.target = target;
+        threadSupport.addCoroutine(this, -1);
+    }
 
     /**
      * Allocates a new {@code Coroutine} object.
+	 * 
      * @param stacksize the size of stack
      */
-	public Coroutine(long stacksize) {
-		this.target = null;
-		threadSupport.addCoroutine(this, stacksize);
-	}
+    public Coroutine(long stacksize) {
+        this.target = null;
+        threadSupport.addCoroutine(this, stacksize);
+    }
 
     /**
      * Allocates a new {@code Coroutine} object.
+	 * 
      * @param target runnable
      * @param stacksize the size of stack
      */
-	public Coroutine(Runnable target, long stacksize) {
-		this.target = target;
-		threadSupport.addCoroutine(this, stacksize);
-	}
+    public Coroutine(Runnable target, long stacksize) {
+        this.target = target;
+        threadSupport.addCoroutine(this, stacksize);
+    }
 
-	/**
+    /**
      * Creates the initial coroutine for a new thread
+	 * 
      * @param threadSupport the CoroutineSupport
-     * @param data the value
+     * @param data 			the value
+     */
+    Coroutine(CoroutineSupport threadSupport, long data) {
+        super(threadSupport, data);
+        this.target = null;
+    }
+
+    /**
+     * Yields execution to the next coroutine in the current threads coroutine queue.
      */
-	Coroutine(CoroutineSupport threadSupport, long data) {
-		super(threadSupport, data);
-		this.target = null;
-	}
-
-	/**
-	 * Yields execution to the next coroutine in the current threads coroutine queue.
-	 */
-	public static void yield() {
-		Thread.currentThread().getCoroutineSupport().symmetricYield();
-	}
-
-	/**
-	 * Yields execution to the other coroutine.
+    public static void yield() {
+        SharedSecrets.getJavaLangAccess().currentThread0().getCoroutineSupport().symmetricYield();
+    }
+
+    /**
+     * Yields execution to the target coroutine.
+     *
      * @param target Coroutine
-	 */
-	public static void yieldTo(Coroutine target) {
-		Thread.currentThread().getCoroutineSupport().symmetricYieldTo(target);
-	}
-
-	/**
-	 * Coroutine exit.
-	 */
-	public void stop() {
-		Thread.currentThread().getCoroutineSupport().symmetricStopCoroutine(this);
-	}
+     */
+    public static void yieldTo(Coroutine target) {
+        SharedSecrets.getJavaLangAccess().currentThread0().getCoroutineSupport().symmetricYieldTo(target);
+    }
+
+    /**
+     * Steal a coroutine from it's carrier thread to current thread.
+     *
+     * @param failOnContention steal fail if there's too much lock contention
+     * @return if stealing succeeds. Also return true directly if coroutine's carrier thread is current.
+     */
+    public boolean steal(boolean failOnContention) {
+        return threadSupport.steal(this, failOnContention);
+    }
+
+    /**
+     * Coroutine exit.
+     */
+    public void stop() {
+        SharedSecrets.getJavaLangAccess().currentThread0().getCoroutineSupport().symmetricStopCoroutine(this);
+    }
 
     /**
-     * the entry to run the coroutine
+     * Relate Coroutine to wisp data structures
+     *
+     * @param id     wispTask id
+     * @param task   task oop
+     * @param engine wispEngine oop
      */
-	protected void run() {
-		assert Thread.currentThread() == threadSupport.getThread();
-		if (target != null) {
-			target.run();
-		}
-	}
+    public void setWispTask(int id, Object task, Object engine) {
+        setWispTask(data, id, task, engine);
+    }
+
+    protected void run() {
+        assert Thread.currentThread() == threadSupport.getThread();
+        if (target != null) {
+            target.run();
+        }
+    }
+
+    static {
+        registerNatives();
+    }
+
+    private static native void registerNatives();
+
+    private static native void setWispTask(long coroutine, int id, Object task, Object engine);
 }
diff --git a/src/java.base/share/classes/java/dyn/CoroutineBase.java b/src/java.base/share/classes/java/dyn/CoroutineBase.java
index 65159797d30..7ee5f4facad 100644
--- a/src/java.base/share/classes/java/dyn/CoroutineBase.java
+++ b/src/java.base/share/classes/java/dyn/CoroutineBase.java
@@ -24,80 +24,90 @@
  */
 
 package java.dyn;
+
+import jdk.internal.access.SharedSecrets;
+
 /**
- *  * Abstract of coroutines.
+ * * Abstract of coroutines.
  */
 public abstract class CoroutineBase {
-	transient long data;
+    transient long data;
 
-	boolean finished = false;
+    boolean finished = false;
 
-	transient CoroutineSupport threadSupport;
+    boolean needsUnlock = false;
+
+    transient CoroutineSupport threadSupport;
 
     /**
-     *  Allocates a new {@code CoroutineBase} object.
+     * Allocates a new {@code CoroutineBase} object.
      */
-	CoroutineBase() {
-		Thread thread = Thread.currentThread();
-		assert thread.getCoroutineSupport() != null;
-		this.threadSupport = thread.getCoroutineSupport();
-	}
+    CoroutineBase() {
+        Thread thread = SharedSecrets.getJavaLangAccess().currentThread0();
+        assert thread.getCoroutineSupport() != null;
+        this.threadSupport = thread.getCoroutineSupport();
+    }
 
-	/**
+    /**
      * Creates the initial coroutine for a new thread
+     *
      * @param threadSupport CoroutineSupport
-     * @param data value
+     * @param data          value
      */
-	CoroutineBase(CoroutineSupport threadSupport, long data) {
-		this.threadSupport = threadSupport;
-		this.data = data;
-	}
+    CoroutineBase(CoroutineSupport threadSupport, long data) {
+        this.threadSupport = threadSupport;
+        this.data = data;
+    }
 
     /**
-     * abstract for the entry of coroutine
+     * entry of coroutine
      */
-	protected abstract void run();
+    protected abstract void run();
 
-	@SuppressWarnings({ "unused" })
-	private final void startInternal() {
-		assert threadSupport.getThread() == Thread.currentThread();
-		try {
-			if (CoroutineSupport.DEBUG) {
-				System.out.println("starting coroutine " + this);
-			}
-			run();
-		} catch (Throwable t) {
-			if (!(t instanceof CoroutineExitException)) {
-				t.printStackTrace();
-			}
-		} finally {
-			finished = true;
-			// use Thread.currentThread().getCoroutineSupport() because we might have been migrated to another thread!
-			Thread.currentThread().getCoroutineSupport().terminateCoroutine();
-		}
-		assert threadSupport.getThread() == Thread.currentThread();
-	}
+    @SuppressWarnings({"unused"})
+    private final void startInternal() {
+        assert threadSupport.getThread() == SharedSecrets.getJavaLangAccess().currentThread0();
+        try {
+            if (false && CoroutineSupport.DEBUG) {
+                System.out.println("starting coroutine " + this);
+            }
+            // When we symmetricYieldTo a newly created coroutine,
+            // we'll expect the new coroutine release lock as soon as possible
+            threadSupport.beforeResume(this);
+            run();
+        } catch (Throwable t) {
+            if (!(t instanceof CoroutineExitException)) {
+                t.printStackTrace();
+            }
+        } finally {
+            finished = true;
+            // threadSupport is fixed by steal()
+            threadSupport.beforeResume(this);
 
-	/**
-	 * @return true if this coroutine has reached its end. Under normal circumstances this happens when the {@link #run()} method returns.
+            threadSupport.terminateCoroutine();
+        }
+        assert threadSupport.getThread() == SharedSecrets.getJavaLangAccess().currentThread0();
+    }
+
+    /**
+     * @return true if this coroutine has reached its end. Under normal circumstances this happens when the {@link #run()} method returns.
      */
-	public final boolean isFinished() {
-		return finished;
-	}
+    public final boolean isFinished() {
+        return finished;
+    }
 
-	/**
-	 * @return the thread that this coroutine is associated with
-	 * @throws NullPointerException
-	 *             if the coroutine has terminated
-	 */
-	public Thread getThread() {
-		return threadSupport.getThread();
-	}
+    /**
+     * @return the thread that this coroutine is associated with
+     * @throws NullPointerException if the coroutine has terminated
+     */
+    public Thread getThread() {
+        return threadSupport.getThread();
+    }
 
     /**
      * @return the current coroutine in the thread
      */
-	public static CoroutineBase current() {
-		return Thread.currentThread().getCoroutineSupport().getCurrent();
-	}
+    public static CoroutineBase current() {
+        return SharedSecrets.getJavaLangAccess().currentThread0().getCoroutineSupport().getCurrent();
+    }
 }
diff --git a/src/java.base/share/classes/java/dyn/CoroutineExitException.java b/src/java.base/share/classes/java/dyn/CoroutineExitException.java
index 19929ac6ecb..e8372d15dd6 100644
--- a/src/java.base/share/classes/java/dyn/CoroutineExitException.java
+++ b/src/java.base/share/classes/java/dyn/CoroutineExitException.java
@@ -29,36 +29,39 @@
  * Implementation of exeception for coroutine exit
  */
 public class CoroutineExitException extends RuntimeException {
-	private static final long serialVersionUID = -2651365020938997924L;
+    private static final long serialVersionUID = -2651365020938997924L;
 
     /**
      * Allocates a new {@code CoroutineExitException} object
      */
-	public CoroutineExitException() {
-	}
+    public CoroutineExitException() {
+    }
 
     /**
      * Allocates a new {@code CoroutineExitException} object
+     *
      * @param message String
-     * @param cause Throwable
+     * @param cause   Throwable
      */
-	public CoroutineExitException(String message, Throwable cause) {
-		super(message, cause);
-	}
+    public CoroutineExitException(String message, Throwable cause) {
+        super(message, cause);
+    }
 
     /**
      * Allocates a new {@code CoroutineExitException} object
+     *
      * @param message String
      */
-	public CoroutineExitException(String message) {
-		super(message);
-	}
+    public CoroutineExitException(String message) {
+        super(message);
+    }
 
     /**
      * Allocates a new {@code CoroutineExitException} object
+     *
      * @param cause Throwable
      */
-	public CoroutineExitException(Throwable cause) {
-		super(cause);
-	}
+    public CoroutineExitException(Throwable cause) {
+        super(cause);
+    }
 }
diff --git a/src/java.base/share/classes/java/dyn/CoroutineSupport.java b/src/java.base/share/classes/java/dyn/CoroutineSupport.java
index 7e7fe934e6f..12cc3053935 100644
--- a/src/java.base/share/classes/java/dyn/CoroutineSupport.java
+++ b/src/java.base/share/classes/java/dyn/CoroutineSupport.java
@@ -26,47 +26,62 @@
 package java.dyn;
 
 import jdk.internal.vm.annotation.IntrinsicCandidate;
+import jdk.internal.access.SharedSecrets;
 import sun.reflect.generics.reflectiveObjects.NotImplementedException;
 
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicReferenceFieldUpdater;
+
 /**
- * The implementation of Coroutine support
+ * Jvm entry of coroutine APIs.
  */
 public class CoroutineSupport {
-	// Controls debugging and tracing, for maximum performance the actual if(DEBUG/TRACE) code needs to be commented out
-	static final boolean DEBUG = false;
-	static final boolean TRACE = false;
+    /*
+     Controls debugging and tracing, for maximum performance the actual if(DEBUG/TRACE) code needs to be commented out
+     the inner of synchronized System.out.println() may result in coroutine switch
+     DO NOT enable DEBUG, TRACE flag before we have a solution.
+      */
+    static final boolean DEBUG = false;
+    static final boolean TRACE = false;
+
+    private static final boolean CHECK_LOCK = true;
+    private static final int SPIN_BACKOFF_LIMIT = 2 << 8;
+
+    private static AtomicInteger idGen = new AtomicInteger();
 
-	static final Object TERMINATED = new Object();
+    // The thread that this CoroutineSupport belongs to. There's only one CoroutineSupport per Thread
+    private final Thread thread;
+    // The initial coroutine of the Thread
+    private final Coroutine threadCoroutine;
 
-	// The thread that this CoroutineSupport belongs to. There's only one CoroutineSupport per Thread
-	private final Thread thread;
-	// The initial coroutine of the Thread
-	private final Coroutine threadCoroutine;
+    // The currently executing coroutine
+    private Coroutine currentCoroutine;
 
-	// The currently executing, symmetric or asymmetric coroutine
-	CoroutineBase currentCoroutine;
-	// The anchor of the doubly-linked ring of coroutines
-	Coroutine scheduledCoroutines;
+    private volatile Thread lockOwner = null; // also protect double link list of JavaThread->coroutine_list()
+    private int lockRecursive; // volatile is not need
 
-	static {
-		registerNatives();
-	}
+    private final int id;
+    private boolean terminated = false;
+
+    static {
+        registerNatives();
+    }
 
     /**
      * Allocates a new {@code CoroutineSupport} object.
      * @param thread the Thread
      */
-	public CoroutineSupport(Thread thread) {
-		if (thread.getCoroutineSupport() != null) {
-			throw new IllegalArgumentException("Cannot instantiate CoroutineThreadSupport for existing Thread");
-		}
-		this.thread = thread;
-		threadCoroutine = new Coroutine(this, getThreadCoroutine());
-		threadCoroutine.next = threadCoroutine;
-		threadCoroutine.last = threadCoroutine;
-		currentCoroutine = threadCoroutine;
-		scheduledCoroutines = threadCoroutine;
-	}
+    public CoroutineSupport(Thread thread) {
+        if (thread.getCoroutineSupport() != null) {
+            throw new IllegalArgumentException("Cannot instantiate CoroutineThreadSupport for existing Thread");
+        }
+        id = idGen.incrementAndGet();
+        this.thread = thread;
+        threadCoroutine = new Coroutine(this, getThreadCoroutine());
+        threadCoroutine.next = threadCoroutine;
+        threadCoroutine.last = threadCoroutine;
+        currentCoroutine = threadCoroutine;
+    }
 
     /**
      * return the threadCoroutine
@@ -76,226 +91,328 @@ public Coroutine threadCoroutine() {
         return threadCoroutine;
     }
 
-    /**
-     * Add a coroutine in coroutine list
-     * @param coroutine Coroutine
-     * @param stacksize the size of stack
-     */
-	void addCoroutine(Coroutine coroutine, long stacksize) {
-		assert scheduledCoroutines != null;
-		assert currentCoroutine != null;
-
-		coroutine.data = createCoroutine(coroutine, stacksize);
-		if (DEBUG) {
-			System.out.println("add Coroutine " + coroutine + ", data" + coroutine.data);
-		}
-
-		// add the coroutine into the doubly linked ring
-		coroutine.next = scheduledCoroutines.next;
-		coroutine.last = scheduledCoroutines;
-		scheduledCoroutines.next = coroutine;
-		coroutine.next.last = coroutine;
-	}
-
-	Thread getThread() {
-		return thread;
-	}
+    void addCoroutine(Coroutine coroutine, long stacksize) {
+        assert currentCoroutine != null;
+        lock();
+        try {
+            coroutine.data = createCoroutine(coroutine, stacksize);
+            if (false && DEBUG) {
+                System.out.println("add Coroutine " + coroutine + ", data" + coroutine.data);
+            }
+
+            // add the coroutine into the doubly linked ring
+            coroutine.next = currentCoroutine.next;
+            coroutine.last = currentCoroutine;
+            currentCoroutine.next = coroutine;
+            coroutine.next.last = coroutine;
+        } finally {
+            unlock();
+        }
+    }
 
-    /**
-     * drain coroutine
-     */
-	public void drain() {
-		if (Thread.currentThread() != thread) {
-			throw new IllegalArgumentException("Cannot drain another threads CoroutineThreadSupport");
-		}
-
-		if (DEBUG) {
-			System.out.println("draining");
-		}
-		try {
-			// drain all scheduled coroutines
-			while (scheduledCoroutines.next != scheduledCoroutines) {
-				symmetricExitInternal(scheduledCoroutines.next);
-			}
-
-			CoroutineBase coro;
-			while ((coro = cleanupCoroutine()) != null) {
-				System.out.println(coro);
-				throw new NotImplementedException();
-			}
-		} catch (Throwable t) {
-			t.printStackTrace();
-		}
-	}
+    Thread getThread() {
+        return thread;
+    }
 
     /**
-     * switch to the next coroutine
+     * drain all alive coroutines.
      */
-	void symmetricYield() {
-		if (scheduledCoroutines != currentCoroutine) {
-			throw new IllegalThreadStateException("Cannot call yield from within an asymmetric coroutine");
-		}
-		assert currentCoroutine instanceof Coroutine;
+    public void drain() {
+        if (Thread.currentThread() != thread) {
+            throw new IllegalArgumentException("Cannot drain another threads CoroutineThreadSupport");
+        }
+
+        if (false && DEBUG) {
+            System.out.println("draining");
+        }
+        lock();
+        try {
+            // drain all coroutines
+            while (currentCoroutine.next != currentCoroutine) {
+                symmetricExitInternal(currentCoroutine.next);
+            }
+
+            CoroutineBase coro;
+            while ((coro = cleanupCoroutine()) != null) {
+                System.out.println(coro);
+                throw new NotImplementedException();
+            }
+        } catch (Throwable t) {
+            t.printStackTrace();
+        } finally {
+            assert lockOwner == thread && lockRecursive == 0;
+            terminated = true;
+            unlock();
+        }
+    }
 
-		if (TRACE) {
-			System.out.println("locking for symmetric yield...");
-		}
+    void symmetricYield() {
+        if (false && TRACE) {
+            System.out.println("locking for symmetric yield...");
+        }
 
-		Coroutine next = scheduledCoroutines.next;
-		if (next == scheduledCoroutines) {
-			return;
-		}
+        lock();
+        Coroutine next = currentCoroutine.next;
+        if (next == currentCoroutine) {
+            unlock();
+            return;
+        }
 
-		if (TRACE) {
-			System.out.println("symmetric yield to " + next);
-		}
+        if (false && TRACE) {
+            System.out.println("symmetric yield to " + next);
+        }
 
-		final Coroutine current = scheduledCoroutines;
-		scheduledCoroutines = next;
-		currentCoroutine = next;
+        final Coroutine current = currentCoroutine;
+        currentCoroutine = next;
 
-		switchTo(current, next);
-	}
+        unlockLater(next);
+        switchTo(current, next);
+        beforeResume(current);
+    }
 
-    /**
-     * switch to the the coroutine
-     * @param target Coroutine
-     */
-	public void symmetricYieldTo(Coroutine target) {
-		if (scheduledCoroutines != currentCoroutine) {
-			throw new IllegalThreadStateException("Cannot call yield from within an asymmetric coroutine");
-		}
-		assert currentCoroutine instanceof Coroutine;
+    void symmetricYieldTo(Coroutine target) {
+        lock();
+        if (target.threadSupport != this) {
+            unlock();
+            return;
+        }
+        moveCoroutine(currentCoroutine, target);
+
+        final Coroutine current = currentCoroutine;
+        currentCoroutine = target;
+        unlockLater(target);
+        switchTo(current, target);
+        beforeResume(current);
+    }
 
-		moveCoroutine(scheduledCoroutines, target);
+    private void moveCoroutine(Coroutine a, Coroutine position) {
+        // remove a from the ring
+        a.last.next = a.next;
+        a.next.last = a.last;
 
-		final Coroutine current = scheduledCoroutines;
-		scheduledCoroutines = target;
-		currentCoroutine = target;
+        // ... and insert at the new position
+        a.next = position.next;
+        a.last = position;
+        a.next.last = a;
+        position.next = a;
+    }
 
-		switchTo(current, target);
-	}
+    void symmetricStopCoroutine(Coroutine target) {
+        Coroutine current;
+        lock();
+        try {
+            if (target.threadSupport != this) {
+                unlock();
+                return;
+            }
+            moveCoroutine(currentCoroutine, target);
+
+            current = currentCoroutine;
+            currentCoroutine = target;
+        } finally {
+            unlock();
+        }
+        switchToAndExit(current, target);
+    }
 
     /**
-     * change the coroutine list
-     * @param a coroutine
-     * @param position coroutine
+     * switch to coroutine and throw Exception in coroutine
      */
-	private void moveCoroutine(Coroutine a, Coroutine position) {
-		// remove a from the ring
-		a.last.next = a.next;
-		a.next.last = a.last;
-
-		// ... and insert at the new position
-		a.next = position.next;
-		a.last = position;
-		a.next.last = a;
-		position.next = a;
-	}
+    private void symmetricExitInternal(Coroutine coroutine) {
+        assert currentCoroutine != coroutine;
+        assert coroutine.threadSupport == this;
+
+        // remove the coroutine from the ring
+        coroutine.last.next = coroutine.next;
+        coroutine.next.last = coroutine.last;
+
+        if (!testDisposableAndTryReleaseStack(coroutine.data)) {
+            // and insert it before the current coroutine
+            coroutine.last = currentCoroutine.last;
+            coroutine.next = currentCoroutine;
+            coroutine.last.next = coroutine;
+            currentCoroutine.last = coroutine;
+
+            final Coroutine current = currentCoroutine;
+            currentCoroutine = coroutine;
+            switchToAndExit(current, coroutine);
+            beforeResume(current);
+        }
+    }
 
     /**
-     * switch to other coroutine, and stop the current.
-     * @param target Coroutine
+     * terminate current coroutine and yield forward
      */
-	public void symmetricStopCoroutine(Coroutine target) {
-		if (scheduledCoroutines != currentCoroutine) {
-			throw new IllegalThreadStateException("Cannot call yield from within an asymmetric coroutine");
-		}
-		assert currentCoroutine instanceof Coroutine;
-
-		moveCoroutine(scheduledCoroutines, target);
-
-		final Coroutine current = scheduledCoroutines;
-		scheduledCoroutines = target;
-		currentCoroutine = target;
-
-		switchToAndExit(current, target);
-	}
+    void terminateCoroutine() {
+        assert currentCoroutine != threadCoroutine : "cannot exit thread coroutine";
+        assert currentCoroutine != currentCoroutine.next : "last coroutine shouldn't call coroutineexit";
+
+        lock();
+        Coroutine old = currentCoroutine;
+        Coroutine forward = old.next;
+        currentCoroutine = forward;
+        old.last.next = old.next;
+        old.next.last = old.last;
+
+        if (false && DEBUG) {
+            System.out.println("to be terminated: " + old);
+        }
+        unlockLater(forward);
+        switchToAndTerminate(old, forward);
+    }
 
     /**
-     * stop the coroutine
-     * @param coroutine Coroutine
+     * Steal coroutine from it's carrier thread to current thread.
+     *
+     * @param failOnContention steal fail if there's too much lock contention
+     * @param coroutine        to be stolen
      */
-	void symmetricExitInternal(Coroutine coroutine) {
-		if (scheduledCoroutines != currentCoroutine) {
-			throw new IllegalThreadStateException("Cannot call exitNext from within an unscheduled coroutine");
-		}
-		assert currentCoroutine instanceof Coroutine;
-		assert currentCoroutine != coroutine;
-
-		// remove the coroutine from the ring
-		coroutine.last.next = coroutine.next;
-		coroutine.next.last = coroutine.last;
-
-		if (!isDisposable(coroutine.data)) {
-			// and insert it before the current coroutine
-			coroutine.last = scheduledCoroutines.last;
-			coroutine.next = scheduledCoroutines;
-			coroutine.last.next = coroutine;
-			scheduledCoroutines.last = coroutine;
-
-			final Coroutine current = scheduledCoroutines;
-			scheduledCoroutines = coroutine;
-			currentCoroutine = coroutine;
-			switchToAndExit(current, coroutine);
-		}
-	}
+    boolean steal(Coroutine coroutine, boolean failOnContention) {
+        assert coroutine.threadSupport.threadCoroutine() != coroutine;
+        CoroutineSupport source = this;
+        CoroutineSupport target = SharedSecrets.getJavaLangAccess().currentThread0().getCoroutineSupport();
+
+        if (source == target) {
+            return true;
+        }
+
+        if (source.id < target.id) { // prevent dead lock
+            if (!source.lockInternal(failOnContention)) {
+                return false;
+            }
+            target.lock();
+        } else {
+            target.lock();
+            if (!source.lockInternal(failOnContention)) {
+                target.unlock();
+                return false;
+            }
+        }
+
+        try {
+            try {
+                if (source.terminated || coroutine.finished ||
+                        coroutine.threadSupport != source || // already been stolen
+                        source.currentCoroutine == coroutine || // running
+                        !stealCoroutine(coroutine.data)) { // native frame
+                    return false;
+                }
+
+                coroutine.last.next = coroutine.next;
+                coroutine.next.last = coroutine.last;
+                coroutine.threadSupport = target;
+            } finally {
+                source.unlock();
+            }
+            coroutine.next = target.currentCoroutine.next;
+            coroutine.last = target.currentCoroutine;
+            target.currentCoroutine.next = coroutine;
+            coroutine.next.last = coroutine;
+        } finally {
+            target.unlock();
+        }
+
+        return true;
+    }
 
     /**
-     * terminate the current coroutine
+     * Can not be stolen while executing this, because lock is held
      */
-	void terminateCoroutine() {
-		assert currentCoroutine == scheduledCoroutines;
-		assert currentCoroutine != threadCoroutine : "cannot exit thread coroutine";
-		assert scheduledCoroutines != scheduledCoroutines.next : "last coroutine shouldn't call coroutineexit";
-
-		Coroutine old = scheduledCoroutines;
-		Coroutine forward = old.next;
-		currentCoroutine = forward;
-		scheduledCoroutines = forward;
-		old.last.next = old.next;
-		old.next.last = old.last;
-
-		if (DEBUG) {
-			System.out.println("to be terminated: " + old);
-		}
-		switchToAndTerminate(old, forward);
-	}
+    void beforeResume(CoroutineBase source) {
+        if (source.needsUnlock) {
+            source.needsUnlock = false;
+            source.threadSupport.unlock();
+        }
+    }
+
+    private void unlockLater(CoroutineBase next) {
+        if (CHECK_LOCK && next.needsUnlock) {
+            throw new InternalError("pending unlock");
+        }
+        next.needsUnlock = true;
+    }
+
+    private void lock() {
+        boolean success = lockInternal(false);
+        assert success;
+    }
+
+    private boolean lockInternal(boolean tryingLock) {
+        final Thread th = SharedSecrets.getJavaLangAccess().currentThread0();
+        if (lockOwner == th) {
+            lockRecursive++;
+            return true;
+        }
+        for (int spin = 1; ; ) {
+            if (lockOwner == null && LOCK_UPDATER.compareAndSet(this, null, th)) {
+                return true;
+            }
+            for (int i = 0; i < spin; ) {
+                i++;
+            }
+            if (spin == SPIN_BACKOFF_LIMIT) {
+                if (tryingLock) {
+                    return false;
+                }
+                SharedSecrets.getJavaLangAccess().yield0(); // yield safepoint
+            } else { // back off
+                spin *= 2;
+            }
+        }
+    }
+
+    private void unlock() {
+        if (CHECK_LOCK && SharedSecrets.getJavaLangAccess().currentThread0() != lockOwner) {
+            throw new InternalError("unlock from non-owner thread");
+        }
+        if (lockRecursive > 0) {
+            lockRecursive--;
+        } else {
+            LOCK_UPDATER.lazySet(this, null);
+        }
+    }
+
+    private static final AtomicReferenceFieldUpdater LOCK_UPDATER;
+
+    static {
+        LOCK_UPDATER = AtomicReferenceFieldUpdater.newUpdater(CoroutineSupport.class, Thread.class, "lockOwner");
+    }	
 
     /**
-     * check whether the coroutine is the current
-     * @param coroutine CoroutineBase
-     * @return true if it is current
+     * @return current running coroutine
      */
-	public boolean isCurrent(CoroutineBase coroutine) {
-		return coroutine == currentCoroutine;
-	}
+    public CoroutineBase getCurrent() {
+        return currentCoroutine;
+    }
+
+    private static native void registerNatives();
+
+    private static native long getThreadCoroutine();
 
     /**
-     * get the current coroutine
-     * @return the current coroutine
+     * need lock because below methods will operate on thread->coroutine_list()
      */
-	public CoroutineBase getCurrent() {
-		return currentCoroutine;
-	}
+    private static native long createCoroutine(CoroutineBase coroutine, long stacksize);
 
-	private static native void registerNatives();
+    @IntrinsicCandidate
+    private static native void switchToAndTerminate(CoroutineBase current, CoroutineBase target);
 
-	private static native long getThreadCoroutine();
+    private static native boolean testDisposableAndTryReleaseStack(long coroutine);
 
-	private static native long createCoroutine(CoroutineBase coroutine, long stacksize);
+    private static native boolean stealCoroutine(long coroPtr);
+    // end of locking
 
     @IntrinsicCandidate
 	private static native void switchTo(CoroutineBase current, CoroutineBase target);
 
     @IntrinsicCandidate
-	private static native void switchToAndTerminate(CoroutineBase current, CoroutineBase target);
-
-    @IntrinsicCandidate
-	private static native void switchToAndExit(CoroutineBase current, CoroutineBase target);
+    private static native void switchToAndExit(CoroutineBase current, CoroutineBase target);
 
-	private static native boolean isDisposable(long coroutine);
+    private static native CoroutineBase cleanupCoroutine();
 
-	private static native CoroutineBase cleanupCoroutine();
+    /**
+     * Telling jvm that wisp is ready to be used.
+     */
+    public static native void setWispBooted();
 
 }
diff --git a/src/java.base/share/classes/java/lang/System.java b/src/java.base/share/classes/java/lang/System.java
index 74cc41870ee..a592f109caa 100644
--- a/src/java.base/share/classes/java/lang/System.java
+++ b/src/java.base/share/classes/java/lang/System.java
@@ -66,6 +66,8 @@
 import java.util.function.Supplier;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.stream.Stream;
+import com.alibaba.wisp.engine.WispEngine;
+import com.alibaba.wisp.engine.WispTask;
 import jdk.internal.misc.Unsafe;
 import jdk.internal.util.StaticProperty;
 import jdk.internal.module.ModuleBootstrap;
@@ -2451,6 +2453,56 @@ public long findNative(ClassLoader loader, String entry) {
             public void exit(int statusCode) {
                 Shutdown.exit(statusCode);
             }
+
+            @Override
+            public Thread currentThread0() {
+                return Thread.currentThread0();
+            }
+
+            @Override
+            public void yield0() {
+                Thread.yield0();
+            }
+
+            @Override
+            public void setWispEngine(Thread thread, WispEngine engine) {
+                thread.wispEngine = engine;
+            }
+
+            @Override
+            public WispEngine getWispEngine(Thread thread) {
+                return thread.wispEngine;
+            }
+
+            @Override
+            public void setWispTask(Thread thread, WispTask task) {
+                thread.wispTask = task;
+            }
+
+            @Override
+            public WispTask getWispTask(Thread thread) {
+                return thread.wispTask;
+            }
+
+            @Override
+            public void setWispAlive(Thread thread, boolean b) {
+                thread.wispIsAlive = b;
+            }
+
+            @Override
+            public boolean isInNative(Thread thread) {
+                return thread.isInNative();
+            }
+
+            @Override
+            public void threadExit(Thread thread) {
+                thread.exit();
+            }
+
+            @Override
+            public void wispBooted() {
+                Thread.wispBooted();
+            }
         });
     }
 }
diff --git a/src/java.base/share/classes/java/lang/Thread.java b/src/java.base/share/classes/java/lang/Thread.java
index d5dd2a90fe2..bb22af09a8d 100644
--- a/src/java.base/share/classes/java/lang/Thread.java
+++ b/src/java.base/share/classes/java/lang/Thread.java
@@ -43,6 +43,12 @@
 import jdk.internal.reflect.CallerSensitive;
 import jdk.internal.reflect.Reflection;
 import jdk.internal.vm.annotation.IntrinsicCandidate;
+import com.alibaba.wisp.engine.WispEngine;
+import com.alibaba.wisp.engine.WispTask;
+import jdk.internal.access.SharedSecrets;
+import jdk.internal.access.WispEngineAccess;
+import jdk.internal.misc.TerminatingThreadLocal;
+import jdk.internal.misc.VM;
 import sun.nio.ch.Interruptible;
 import sun.security.util.SecurityConstants;
 
@@ -196,6 +202,11 @@ private static synchronized int nextThreadNum() {
      */
     private final long stackSize;
 
+    /*
+     * JVM-private state that persists after native thread termination.
+     */
+    private long nativeParkEventPointer;
+
     /*
      * Thread ID
      */
@@ -258,29 +269,70 @@ static void blockedOn(Interruptible b) {
      */
     private CoroutineSupport coroutineSupport;
 
+    WispEngine wispEngine;
+
+    WispTask wispTask;
+
+    volatile boolean wispIsAlive;
+
     /**
      * @return the coroutine support of the thread
      */
     public CoroutineSupport getCoroutineSupport() {
-        return coroutineSupport;
+        if (coroutineSupport != null) {
+            return coroutineSupport;
+        }
+
+        Thread t = currentThread0();
+        return t == this ? null : t.getCoroutineSupport();
     }
 
     /**
      * initialize the coroutine support of the thread
      */
     private void initializeCoroutineSupport() {
-        if (jdk.internal.misc.VM.isEnableCoroutine()) {
+        if (coroutineSupport == null) {
             coroutineSupport = new CoroutineSupport(this);
         }
     }
 
+    private void destroyCoroutineSupport() {
+        try {
+            if (coroutineSupport != null) {
+                if (WEA != null) {
+                    WEA.eventLoop();
+                    WEA.destroy();
+                }
+                coroutineSupport.drain();
+            }
+        } catch (Throwable t) {
+            t.printStackTrace();
+        }
+    }
+
     /**
      * Returns a reference to the currently executing thread object.
      *
+     * Returns a thread wrapper of the currently executing wispTask if
+     * WispEngine.transparentWispSwitch() is on.
+     * 
      * @return  the currently executing thread.
      */
+    public static Thread currentThread() {
+        if (WEA != null) {
+            return WEA.getCurrentTask().getThreadWrapper();
+        }
+        return currentThread0();
+    }
+
+    // use local copy to reduce test branch
+    private static boolean vmBooted = false;
+
+    /**
+     * Always return the underlying Java thread.
+     */
     @IntrinsicCandidate
-    public static native Thread currentThread();
+    static native Thread currentThread0();
 
     /**
      * A hint to the scheduler that the current thread is willing to yield
@@ -298,7 +350,19 @@ private void initializeCoroutineSupport() {
      * concurrency control constructs such as the ones in the
      * {@link java.util.concurrent.locks} package.
      */
-    public static native void yield();
+    public static void yield() {
+        if (WEA != null) {
+            WEA.yield();
+        } else {
+            yield0();
+        }
+    }
+
+    /**
+     * Always use the thread semantics.
+     */
+    static native void yield0();
+
 
     /**
      * Causes the currently executing thread to sleep (temporarily cease
@@ -317,7 +381,20 @@ private void initializeCoroutineSupport() {
      *          interrupted status of the current thread is
      *          cleared when this exception is thrown.
      */
-    public static native void sleep(long millis) throws InterruptedException;
+    public static void sleep(long millis) throws InterruptedException {
+        if (WEA != null) {
+            if (Thread.interrupted()) {
+                throw new InterruptedException("sleep interrupted");
+            }
+            WEA.sleep(millis);
+            if (Thread.interrupted()) {
+                throw new InterruptedException("sleep interrupted");
+            }
+            return;
+        } else {
+            sleep0(millis);
+        }
+    }
 
     /**
      * Causes the currently executing thread to sleep (temporarily cease
@@ -359,6 +436,11 @@ public static void sleep(long millis, int nanos)
         sleep(millis);
     }
 
+    /**
+     * Always block the native thread
+     */
+    private static native void sleep0(long millis) throws InterruptedException;
+
     /**
      * Indicates that the caller is momentarily unable to progress, until the
      * occurrence of one or more actions on the part of other activities. By
@@ -821,7 +903,10 @@ public synchronized void start() {
 
         boolean started = false;
         try {
-            start0();
+            if (!(WEA != null && WispEngine.enableThreadAsWisp() &&
+                    WEA.tryStartThreadAsWisp(this, target))) {
+                start0();
+            }
             started = true;
         } finally {
             try {
@@ -860,13 +945,10 @@ public void run() {
      * This method is called by the system to give a Thread
      * a chance to clean up before it actually exits.
      */
-    private void exit() {
+    void exit() {
         if (threadLocals != null && TerminatingThreadLocal.REGISTRY.isPresent()) {
             TerminatingThreadLocal.threadTerminated();
         }
-        if (jdk.internal.misc.VM.isEnableCoroutine() && (coroutineSupport != null)) {
-            coroutineSupport.drain();
-        }
         if (group != null) {
             group.threadTerminated(this);
             group = null;
@@ -1018,15 +1100,22 @@ public void interrupt() {
                 Interruptible b = blocker;
                 if (b != null) {
                     interrupted = true;
-                    interrupt0();  // inform VM of interrupt
+                    if (WEA != null && wispTask != null) {
+                        WEA.interrupt(wispTask);
+                    } else {
+                        interrupt0();           // Just to set the interrupt flag
+                    }
                     b.interrupt(this);
                     return;
                 }
             }
         }
         interrupted = true;
-        // inform VM of interrupt
-        interrupt0();
+        if (WEA != null && wispTask != null) {
+            WEA.interrupt(wispTask);
+        } else {
+            interrupt0();
+        }
     }
 
     /**
@@ -1043,13 +1132,19 @@ public void interrupt() {
      * @revised 6.0, 14
      */
     public static boolean interrupted() {
-        Thread t = currentThread();
-        boolean interrupted = t.interrupted;
+        Thread current = currentThread0();
+        if (WEA != null) {
+            WispTask task = WEA.getCurrentTask();
+            if (task != null) {
+                return WEA.testInterruptedAndClear(task, true);
+            }
+        }
+        boolean interrupted = current.interrupted;
         // We may have been interrupted the moment after we read the field,
         // so only clear the field if we saw that it was set and will return
         // true; otherwise we could lose an interrupt.
         if (interrupted) {
-            t.interrupted = false;
+            current.interrupted = false;
             clearInterruptEvent();
         }
         return interrupted;
@@ -1065,6 +1160,9 @@ public static boolean interrupted() {
      * @revised 6.0, 14
      */
     public boolean isInterrupted() {
+        if (WEA != null && wispTask != null) {
+            return WEA.testInterruptedAndClear(wispTask, false);
+        }
         return interrupted;
     }
 
@@ -1075,7 +1173,14 @@ public boolean isInterrupted() {
      * @return  {@code true} if this thread is alive;
      *          {@code false} otherwise.
      */
-    public final native boolean isAlive();
+    public final boolean isAlive() {
+        if (WEA != null && wispIsAlive) {
+            return true;
+        }
+        return isAlive0();
+    }
+
+    private final native boolean isAlive0();
 
     /**
      * Suspends this thread.
@@ -1610,7 +1715,29 @@ public void setContextClassLoader(ClassLoader cl) {
      * @since 1.5
      */
     public StackTraceElement[] getStackTrace() {
-        if (this != Thread.currentThread()) {
+        boolean slowPath;
+        if (WEA != null) {
+            WispTask task = this.wispTask;
+            if (task == null) {
+                // When we create a thread, coroutine will not be created immediately.
+                // So if we take the stack trace at once, it will NCE.
+                // See: NullStackTrace.java and 6571589, Thread return an array which size is 0.
+                return new StackTraceElement[0];
+            }
+            if (!WEA.runningAsCoroutine(this)) {
+                slowPath = this != Thread.currentThread();
+            } else {
+                boolean isCurrentTask = WEA.getCurrentTask() == task;
+                if (!WEA.isThreadTask(task) && !isCurrentTask) {
+                    throw new UnsupportedOperationException("NYI, can NOT get other runnable Coroutine stacktrace!");
+                }
+                slowPath = !isCurrentTask;
+            }
+        } else {
+            slowPath = this != Thread.currentThread();
+        }
+
+        if (slowPath) {
             // check for getStackTrace permission
             @SuppressWarnings("removal")
             SecurityManager security = System.getSecurityManager();
@@ -1891,6 +2018,11 @@ public State getState() {
         return jdk.internal.misc.VM.toThreadState(threadStatus);
     }
 
+    /**
+     * @return if this thread is executing JNI code
+     */
+    native boolean isInNative();
+
     // Added in JSR-166
 
     /**
@@ -2122,4 +2254,15 @@ public boolean equals(Object obj) {
     private native void interrupt0();
     private static native void clearInterruptEvent();
     private native void setNativeName(String name);
+
+    // prevent load Wisp classes accidentally
+    private static WispEngineAccess WEA;
+
+    static void wispBooted() {
+        if (!WispEngine.transparentWispSwitch()) {
+            // assert in this class will crash jvm
+            throw new AssertionError();
+        }
+        WEA = SharedSecrets.getWispEngineAccess();
+    }
 }
diff --git a/src/java.base/share/classes/java/net/DatagramSocket.java b/src/java.base/share/classes/java/net/DatagramSocket.java
index db3310d040f..020b2e1d69b 100644
--- a/src/java.base/share/classes/java/net/DatagramSocket.java
+++ b/src/java.base/share/classes/java/net/DatagramSocket.java
@@ -25,6 +25,9 @@
 
 package java.net;
 
+import com.alibaba.wisp.engine.WispEngine;
+import sun.nio.ch.WispUdpSocketImpl;
+
 import java.io.IOException;
 import java.io.UncheckedIOException;
 import java.nio.channels.DatagramChannel;
@@ -247,6 +250,8 @@
  */
 public class DatagramSocket implements java.io.Closeable {
 
+    private WispUdpSocketImpl asyncImpl;
+
     // An instance of DatagramSocketAdaptor, NetMulticastSocket, or null
     private final DatagramSocket delegate;
 
@@ -327,6 +332,11 @@ protected DatagramSocket(DatagramSocketImpl impl) {
      * @since   1.4
      */
     public DatagramSocket(SocketAddress bindaddr) throws SocketException {
+        // if (WispEngine.transparentWispSwitch()) {
+        //     asyncImpl = new WispUdpSocketImpl(this);
+        // } else {
+        //     DatagramSocket(createDelegate(bindaddr, DatagramSocket.class));
+        // }
         this(createDelegate(bindaddr, DatagramSocket.class));
     }
 
@@ -405,6 +415,10 @@ public DatagramSocket(int port, InetAddress laddr) throws SocketException {
      * @since 1.4
      */
     public void bind(SocketAddress addr) throws SocketException {
+        if (WispEngine.transparentWispSwitch()) {
+            asyncImpl.bind(addr);
+            return;
+        }
         delegate().bind(addr);
     }
 
@@ -471,6 +485,10 @@ public void bind(SocketAddress addr) throws SocketException {
      * @since 1.2
      */
     public void connect(InetAddress address, int port) {
+        if (WispEngine.transparentWispSwitch()) {
+            asyncImpl.connect(address, port);
+            return;
+        }
         delegate().connect(address, port);
     }
 
@@ -502,6 +520,10 @@ public void connect(InetAddress address, int port) {
      * @since 1.4
      */
     public void connect(SocketAddress addr) throws SocketException {
+        if (WispEngine.transparentWispSwitch()) {
+            asyncImpl.connect(addr);
+            return;
+        }
         delegate().connect(addr);
     }
 
@@ -523,6 +545,10 @@ public void connect(SocketAddress addr) throws SocketException {
      * @since 1.2
      */
     public void disconnect() {
+        if (WispEngine.transparentWispSwitch()) {
+            asyncImpl.disconnect();
+            return;
+        }
         delegate().disconnect();
     }
 
@@ -537,6 +563,9 @@ public void disconnect() {
      * @since 1.4
      */
     public boolean isBound() {
+        if (WispEngine.transparentWispSwitch()) {
+            return asyncImpl.isBound();
+        }
         return delegate().isBound();
     }
 
@@ -551,6 +580,9 @@ public boolean isBound() {
      * @since 1.4
      */
     public boolean isConnected() {
+        if (WispEngine.transparentWispSwitch()) {
+            return asyncImpl.isConnected();
+        }
         return delegate().isConnected();
     }
 
@@ -566,6 +598,9 @@ public boolean isConnected() {
      * @since 1.2
      */
     public InetAddress getInetAddress() {
+        if (WispEngine.transparentWispSwitch()) {
+            return asyncImpl.getInetAddress();
+        }
         return delegate().getInetAddress();
     }
 
@@ -581,6 +616,9 @@ public InetAddress getInetAddress() {
      * @since 1.2
      */
     public int getPort() {
+        if (WispEngine.transparentWispSwitch()) {
+            return asyncImpl.getPort();
+        }
         return delegate().getPort();
     }
 
@@ -661,6 +699,10 @@ public SocketAddress getLocalSocketAddress() {
      * @revised 1.4
      */
     public void send(DatagramPacket p) throws IOException  {
+        if (WispEngine.transparentWispSwitch()) {
+            asyncImpl.send(p);
+            return;
+        }
         delegate().send(p);
     }
 
@@ -697,6 +739,10 @@ public void send(DatagramPacket p) throws IOException  {
      * @revised 1.4
      */
     public void receive(DatagramPacket p) throws IOException {
+        if (WispEngine.transparentWispSwitch()) {
+            p.length = asyncImpl.receive(p, p.bufLength);
+            return;
+        }
         delegate().receive(p);
     }
 
@@ -719,6 +765,9 @@ public void receive(DatagramPacket p) throws IOException {
      * @since   1.1
      */
     public InetAddress getLocalAddress() {
+        if (WispEngine.transparentWispSwitch()) {
+            return asyncImpl.getLocalAddress();
+        }
         return delegate().getLocalAddress();
     }
 
@@ -731,6 +780,9 @@ public InetAddress getLocalAddress() {
      *          {@code 0} if it is not bound yet.
      */
     public int getLocalPort() {
+        if (WispEngine.transparentWispSwitch()) {
+            return asyncImpl.getLocalPort();
+        }
         return delegate().getLocalPort();
     }
 
@@ -752,6 +804,10 @@ public int getLocalPort() {
      * @see #getSoTimeout()
      */
     public void setSoTimeout(int timeout) throws SocketException {
+        if (WispEngine.transparentWispSwitch()) {
+            asyncImpl.setSoTimeout(timeout);
+            return;
+        }
         delegate().setSoTimeout(timeout);
     }
 
@@ -765,6 +821,9 @@ public void setSoTimeout(int timeout) throws SocketException {
      * @see #setSoTimeout(int)
      */
     public int getSoTimeout() throws SocketException {
+        if (WispEngine.transparentWispSwitch()) {
+            return asyncImpl.getSoTimeout();
+        }
         return delegate().getSoTimeout();
     }
 
@@ -805,6 +864,10 @@ public int getSoTimeout() throws SocketException {
      * @since 1.2
      */
     public void setSendBufferSize(int size) throws SocketException {
+        if (WispEngine.transparentWispSwitch()) {
+            asyncImpl.setSendBufferSize(size);
+            return;
+        }
         delegate().setSendBufferSize(size);
     }
 
@@ -824,6 +887,9 @@ public void setSendBufferSize(int size) throws SocketException {
      * @since 1.2
      */
     public int getSendBufferSize() throws SocketException {
+        if (WispEngine.transparentWispSwitch()) {
+            return asyncImpl.getSendBufferSize();
+        }
         return delegate().getSendBufferSize();
     }
 
@@ -863,6 +929,10 @@ public int getSendBufferSize() throws SocketException {
      * @since 1.2
      */
     public void setReceiveBufferSize(int size) throws SocketException {
+        if (WispEngine.transparentWispSwitch()) {
+            asyncImpl.setReceiveBufferSize(size);
+            return;
+        }
         delegate().setReceiveBufferSize(size);
     }
 
@@ -881,6 +951,9 @@ public void setReceiveBufferSize(int size) throws SocketException {
      * @since 1.2
      */
     public int getReceiveBufferSize() throws SocketException {
+        if (WispEngine.transparentWispSwitch()) {
+            return asyncImpl.getReceiveBufferSize();
+        }
         return delegate().getReceiveBufferSize();
     }
 
@@ -924,6 +997,10 @@ public int getReceiveBufferSize() throws SocketException {
      * @see StandardSocketOptions#SO_REUSEADDR
      */
     public void setReuseAddress(boolean on) throws SocketException {
+        if (WispEngine.transparentWispSwitch()) {
+            asyncImpl.setReuseAddress(on);
+            return;
+        }
         delegate().setReuseAddress(on);
     }
 
@@ -942,6 +1019,9 @@ public void setReuseAddress(boolean on) throws SocketException {
      * @see StandardSocketOptions#SO_REUSEADDR
      */
     public boolean getReuseAddress() throws SocketException {
+        if (WispEngine.transparentWispSwitch()) {
+            return asyncImpl.getReuseAddress();
+        }
         return delegate().getReuseAddress();
     }
 
@@ -968,6 +1048,10 @@ public boolean getReuseAddress() throws SocketException {
      * @see StandardSocketOptions#SO_BROADCAST
      */
     public void setBroadcast(boolean on) throws SocketException {
+        if (WispEngine.transparentWispSwitch()) {
+            asyncImpl.setBroadcast(on);
+            return;
+        }
         delegate().setBroadcast(on);
     }
 
@@ -986,6 +1070,9 @@ public void setBroadcast(boolean on) throws SocketException {
      * @see StandardSocketOptions#SO_BROADCAST
      */
     public boolean getBroadcast() throws SocketException {
+        if (WispEngine.transparentWispSwitch()) {
+            return asyncImpl.getBroadcast();
+        }
         return delegate().getBroadcast();
     }
 
@@ -1032,6 +1119,10 @@ public boolean getBroadcast() throws SocketException {
      * @see StandardSocketOptions#IP_TOS
      */
     public void setTrafficClass(int tc) throws SocketException {
+        if (WispEngine.transparentWispSwitch()) {
+            asyncImpl.setTrafficClass(tc);
+            return;
+        }
         delegate().setTrafficClass(tc);
     }
 
@@ -1057,6 +1148,9 @@ public void setTrafficClass(int tc) throws SocketException {
      * @see StandardSocketOptions#IP_TOS
      */
     public int getTrafficClass() throws SocketException {
+        if (WispEngine.transparentWispSwitch()) {
+            return asyncImpl.getTrafficClass();
+        }
         return delegate().getTrafficClass();
     }
 
@@ -1072,6 +1166,10 @@ public int getTrafficClass() throws SocketException {
      * @revised 1.4
      */
     public void close() {
+        if (WispEngine.transparentWispSwitch()) {
+            asyncImpl.close();
+            return;
+        }
         delegate().close();
     }
 
@@ -1082,6 +1180,9 @@ public void close() {
      * @since 1.4
      */
     public boolean isClosed() {
+        if (WispEngine.transparentWispSwitch()) {
+            return asyncImpl.isClosed();
+        }
         return delegate().isClosed();
     }
 
@@ -1099,7 +1200,7 @@ public boolean isClosed() {
      * @since 1.4
      */
     public DatagramChannel getChannel() {
-        return null;
+        return WispEngine.transparentWispSwitch() ? asyncImpl.getChannel() : null;
     }
 
     /**
diff --git a/src/java.base/share/classes/java/net/ServerSocket.java b/src/java.base/share/classes/java/net/ServerSocket.java
index 3d985f84df3..ca374f038c2 100644
--- a/src/java.base/share/classes/java/net/ServerSocket.java
+++ b/src/java.base/share/classes/java/net/ServerSocket.java
@@ -25,6 +25,7 @@
 
 package java.net;
 
+import com.alibaba.wisp.engine.WispEngine;
 import java.io.FileDescriptor;
 import java.io.IOException;
 import java.nio.channels.ServerSocketChannel;
@@ -34,6 +35,7 @@
 
 import sun.security.util.SecurityConstants;
 import sun.net.PlatformSocketImpl;
+import sun.nio.ch.WispServerSocketImpl;
 
 /**
  * This class implements server sockets. A server socket waits for
@@ -77,6 +79,9 @@
  * @since   1.0
  */
 public class ServerSocket implements java.io.Closeable {
+
+    private WispServerSocketImpl asyncImpl;
+
     /**
      * Various states of this socket.
      */
@@ -105,6 +110,9 @@ public class ServerSocket implements java.io.Closeable {
     protected ServerSocket(SocketImpl impl) {
         Objects.requireNonNull(impl);
         checkPermission();
+        if (WispEngine.transparentWispSwitch()) {
+            throw new UnsupportedOperationException();
+        }
         this.impl = impl;
     }
 
@@ -124,6 +132,10 @@ private static Void checkPermission() {
      * @revised 1.4
      */
     public ServerSocket() throws IOException {
+        if (WispEngine.transparentWispSwitch()) {
+            asyncImpl = new WispServerSocketImpl();
+            return;
+        }
         setImpl();
     }
 
@@ -264,7 +276,11 @@ public ServerSocket(int port, int backlog) throws IOException {
      * @since   1.1
      */
     public ServerSocket(int port, int backlog, InetAddress bindAddr) throws IOException {
-        setImpl();
+        if (WispEngine.transparentWispSwitch()) {
+            asyncImpl = new WispServerSocketImpl();
+        } else {
+            setImpl();
+        }
         if (port < 0 || port > 0xFFFF)
             throw new IllegalArgumentException(
                        "Port value out of range: " + port);
@@ -290,12 +306,18 @@ public ServerSocket(int port, int backlog, InetAddress bindAddr) throws IOExcept
      * @since 1.4
      */
     SocketImpl getImpl() throws SocketException {
+        if (WispEngine.transparentWispSwitch()) {
+            throw new UnsupportedOperationException();
+        }
         if (!created)
             createImpl();
         return impl;
     }
 
     private void setImpl() {
+        if (WispEngine.transparentWispSwitch()) {
+            throw new UnsupportedOperationException();
+        }
         SocketImplFactory factory = ServerSocket.factory;
         if (factory != null) {
             impl = factory.createSocketImpl();
@@ -311,6 +333,9 @@ private void setImpl() {
      * @since 1.4
      */
     void createImpl() throws SocketException {
+        if (WispEngine.transparentWispSwitch()) {
+            throw new UnsupportedOperationException();
+        }
         if (impl == null)
             setImpl();
         try {
@@ -380,6 +405,10 @@ public void bind(SocketAddress endpoint, int backlog) throws IOException {
             throw new SocketException("Unresolved address");
         if (backlog < 1)
           backlog = 50;
+        if (WispEngine.transparentWispSwitch()) {
+            asyncImpl.bind(endpoint, backlog);
+            return;
+        }
         try {
             @SuppressWarnings("removal")
             SecurityManager security = System.getSecurityManager();
@@ -416,6 +445,9 @@ public void bind(SocketAddress endpoint, int backlog) throws IOException {
      * @see SecurityManager#checkConnect
      */
     public InetAddress getInetAddress() {
+        if (WispEngine.transparentWispSwitch()) {
+            return asyncImpl.getInetAddress();
+        }
         if (!isBound())
             return null;
         try {
@@ -446,6 +478,9 @@ public InetAddress getInetAddress() {
      *          -1 if the socket is not bound yet.
      */
     public int getLocalPort() {
+        if (WispEngine.transparentWispSwitch()) {
+            return asyncImpl.getLocalPort();
+        }
         if (!isBound())
             return -1;
         try {
@@ -528,6 +563,9 @@ public Socket accept() throws IOException {
             throw new SocketException("Socket is closed");
         if (!isBound())
             throw new SocketException("Socket is not bound yet");
+        if (WispEngine.transparentWispSwitch()) {
+            return asyncImpl.accept();
+        }
         Socket s = new Socket((SocketImpl) null);
         implAccept(s);
         return s;
@@ -567,6 +605,9 @@ public Socket accept() throws IOException {
      * @revised 1.4
      */
     protected final void implAccept(Socket s) throws IOException {
+        if (WispEngine.transparentWispSwitch()) {
+            throw new UnsupportedOperationException();
+        }
         SocketImpl si = s.impl;
 
         // Socket has no SocketImpl
@@ -711,6 +752,9 @@ private void ensureCompatible(SocketImpl si) throws IOException {
      * @revised 1.4
      */
     public void close() throws IOException {
+        if (WispEngine.transparentWispSwitch()) {
+            asyncImpl.close();
+        }
         synchronized(closeLock) {
             if (isClosed())
                 return;
@@ -736,7 +780,7 @@ public void close() throws IOException {
      * @since 1.4
      */
     public ServerSocketChannel getChannel() {
-        return null;
+        return WispEngine.transparentWispSwitch() ? asyncImpl.getChannel() : null;
     }
 
     /**
@@ -750,6 +794,9 @@ public ServerSocketChannel getChannel() {
      * @since 1.4
      */
     public boolean isBound() {
+        if (WispEngine.transparentWispSwitch()) {
+            return asyncImpl.isBound();
+        }
         return bound;
     }
 
@@ -760,6 +807,9 @@ public boolean isBound() {
      * @since 1.4
      */
     public boolean isClosed() {
+        if (WispEngine.transparentWispSwitch()) {
+            return asyncImpl.isClosed();
+        }
         synchronized(closeLock) {
             return closed;
         }
@@ -788,6 +838,10 @@ public synchronized void setSoTimeout(int timeout) throws SocketException {
             throw new SocketException("Socket is closed");
         if (timeout < 0)
             throw new IllegalArgumentException("timeout < 0");
+        if (WispEngine.transparentWispSwitch()) {
+            asyncImpl.setSoTimeout(timeout);
+            return;
+        }
         getImpl().setOption(SocketOptions.SO_TIMEOUT, timeout);
     }
 
@@ -802,6 +856,9 @@ public synchronized void setSoTimeout(int timeout) throws SocketException {
     public synchronized int getSoTimeout() throws IOException {
         if (isClosed())
             throw new SocketException("Socket is closed");
+        if (WispEngine.transparentWispSwitch()) {
+            return asyncImpl.getSoTimeout();
+        }
         Object o = getImpl().getOption(SocketOptions.SO_TIMEOUT);
         /* extra type safety */
         if (o instanceof Integer) {
@@ -850,6 +907,10 @@ public synchronized int getSoTimeout() throws IOException {
     public void setReuseAddress(boolean on) throws SocketException {
         if (isClosed())
             throw new SocketException("Socket is closed");
+        if (WispEngine.transparentWispSwitch()) {
+            asyncImpl.setReuseAddress(on);
+            return;
+        }
         getImpl().setOption(SocketOptions.SO_REUSEADDR, Boolean.valueOf(on));
     }
 
@@ -866,6 +927,9 @@ public void setReuseAddress(boolean on) throws SocketException {
     public boolean getReuseAddress() throws SocketException {
         if (isClosed())
             throw new SocketException("Socket is closed");
+        if (WispEngine.transparentWispSwitch()) {
+            return asyncImpl.getReuseAddress();
+        }
         return ((Boolean) (getImpl().getOption(SocketOptions.SO_REUSEADDR))).booleanValue();
     }
 
@@ -885,6 +949,9 @@ public boolean getReuseAddress() throws SocketException {
      */
     @SuppressWarnings("removal")
     public String toString() {
+        if (WispEngine.transparentWispSwitch()) {
+            return asyncImpl.toString();
+        }
         if (!isBound())
             return "ServerSocket[unbound]";
         InetAddress in;
@@ -937,6 +1004,9 @@ public String toString() {
      */
     @Deprecated(since = "17")
     public static synchronized void setSocketFactory(SocketImplFactory fac) throws IOException {
+        if (WispEngine.transparentWispSwitch()) {
+            throw new UnsupportedOperationException();
+        }
         if (factory != null) {
             throw new SocketException("factory already defined");
         }
@@ -990,6 +1060,10 @@ public synchronized void setReceiveBufferSize (int size) throws SocketException
         }
         if (isClosed())
             throw new SocketException("Socket is closed");
+         if (WispEngine.transparentWispSwitch()) {
+             asyncImpl.setReceiveBufferSize(size);
+             return;
+         }
         getImpl().setOption(SocketOptions.SO_RCVBUF, size);
     }
 
@@ -1011,6 +1085,9 @@ public synchronized int getReceiveBufferSize()
     throws SocketException{
         if (isClosed())
             throw new SocketException("Socket is closed");
+        if (WispEngine.transparentWispSwitch()) {
+            return asyncImpl.getReceiveBufferSize();
+        }
         int result = 0;
         Object o = getImpl().getOption(SocketOptions.SO_RCVBUF);
         if (o instanceof Integer) {
@@ -1097,7 +1174,11 @@ public  ServerSocket setOption(SocketOption name, T value)
         Objects.requireNonNull(name);
         if (isClosed())
             throw new SocketException("Socket is closed");
-        getImpl().setOption(name, value);
+        if (WispEngine.transparentWispSwitch()) {
+            getChannel().setOption(name, value);
+        } else {
+            getImpl().setOption(name, value);
+        }
         return this;
     }
 
@@ -1128,7 +1209,7 @@ public  T getOption(SocketOption name) throws IOException {
         Objects.requireNonNull(name);
         if (isClosed())
             throw new SocketException("Socket is closed");
-        return getImpl().getOption(name);
+        return WispEngine.transparentWispSwitch() ? getChannel().getOption(name) : getImpl().getOption(name);
     }
 
     // cache of unmodifiable impl options. Possibly set racy, in impl we trust
diff --git a/src/java.base/share/classes/java/net/Socket.java b/src/java.base/share/classes/java/net/Socket.java
index 967d64ef63b..7b9a6d7769c 100644
--- a/src/java.base/share/classes/java/net/Socket.java
+++ b/src/java.base/share/classes/java/net/Socket.java
@@ -26,6 +26,8 @@
 package java.net;
 
 import sun.security.util.SecurityConstants;
+import com.alibaba.wisp.engine.WispEngine;
+import sun.nio.ch.WispSocketImpl;
 
 import java.io.InputStream;
 import java.io.OutputStream;
@@ -96,6 +98,9 @@
  * @since   1.0
  */
 public class Socket implements java.io.Closeable {
+
+    private WispSocketImpl asyncImpl;
+
     /**
      * Various states of this socket.
      */
@@ -140,9 +145,26 @@ public class Socket implements java.io.Closeable {
      * @revised 1.4
      */
     public Socket() {
+        if (WispEngine.transparentWispSwitch()) {
+            asyncImpl = new WispSocketImpl(this);
+            return;
+        }
         setImpl();
     }
 
+    /**
+     * Created Socket by nio channel.
+     * Only used by wisp.
+     *
+     * @param sc the underlay channel implementation.
+     */
+    public Socket(SocketChannel sc) {
+        if (!WispEngine.transparentWispSwitch()) {
+            throw new UnsupportedOperationException();
+        }
+        asyncImpl = new WispSocketImpl(sc, this);
+    }
+
     /**
      * Creates an unconnected socket, specifying the type of proxy, if any,
      * that should be used regardless of any other settings.
@@ -176,6 +198,14 @@ public Socket(Proxy proxy) {
         if (proxy == null) {
             throw new IllegalArgumentException("Invalid Proxy");
         }
+        if (WispEngine.transparentWispSwitch()) {
+            if (proxy == Proxy.NO_PROXY) {
+                asyncImpl = new WispSocketImpl(this);
+                return;
+            } else {
+                throw new UnsupportedOperationException();
+            }
+        }
         Proxy p = proxy == Proxy.NO_PROXY ? Proxy.NO_PROXY
                                           : sun.net.ApplicationProxy.create(proxy);
         Proxy.Type type = p.type();
@@ -494,14 +524,23 @@ public Socket(InetAddress host, int port, boolean stream) throws IOException {
 
     private Socket(SocketAddress address, SocketAddress localAddr,
                    boolean stream) throws IOException {
-        setImpl();
+        if (WispEngine.transparentWispSwitch()) {
+            if (!stream) {
+                throw new UnsupportedOperationException();
+            }
+            asyncImpl = new WispSocketImpl(this);
+        } else {
+            setImpl();
+        }
 
         // backward compatibility
         if (address == null)
             throw new NullPointerException();
 
         try {
-            createImpl(stream);
+            if (!WispEngine.transparentWispSwitch()) {
+                createImpl(stream);
+            }
             if (localAddr != null)
                 bind(localAddr);
             connect(address);
@@ -524,6 +563,9 @@ private Socket(SocketAddress address, SocketAddress localAddr,
      * @since 1.4
      */
      void createImpl(boolean stream) throws SocketException {
+         if (WispEngine.transparentWispSwitch()) {
+             throw new UnsupportedOperationException();
+         }
         if (impl == null)
             setImpl();
         try {
@@ -543,6 +585,9 @@ void setImpl(SocketImpl si) {
      * @since 1.4
      */
     void setImpl() {
+        if (WispEngine.transparentWispSwitch()) {
+            throw new UnsupportedOperationException();
+        }
         SocketImplFactory factory = Socket.factory;
         if (factory != null) {
             impl = factory.createSocketImpl();
@@ -562,6 +607,9 @@ void setImpl() {
      * @since 1.4
      */
     SocketImpl getImpl() throws SocketException {
+        if (WispEngine.transparentWispSwitch()) {
+            throw new UnsupportedOperationException();
+        }
         if (!created)
             createImpl(true);
         return impl;
@@ -616,6 +664,11 @@ public void connect(SocketAddress endpoint, int timeout) throws IOException {
         if (!(endpoint instanceof InetSocketAddress epoint))
             throw new IllegalArgumentException("Unsupported address type");
 
+        if (WispEngine.transparentWispSwitch()) {
+            asyncImpl.connect(endpoint, timeout);
+            return;
+        }
+
         InetAddress addr = epoint.getAddress ();
         int port = epoint.getPort();
         checkAddress(addr, "connect");
@@ -671,6 +724,11 @@ public void bind(SocketAddress bindpoint) throws IOException {
         if (epoint == null) {
             epoint = new InetSocketAddress(0);
         }
+        if (WispEngine.transparentWispSwitch()) {
+            asyncImpl.bind(epoint);
+            return;
+        }
+
         InetAddress addr = epoint.getAddress();
         int port = epoint.getPort();
         checkAddress (addr, "bind");
@@ -696,6 +754,9 @@ private void checkAddress (InetAddress addr, String op) {
      * set the flags after an accept() call.
      */
     final void postAccept() {
+        if (WispEngine.transparentWispSwitch()) {
+            throw new UnsupportedOperationException();
+        }
         connected = true;
         created = true;
         bound = true;
@@ -714,6 +775,9 @@ final void postAccept() {
     public InetAddress getInetAddress() {
         if (!isConnected())
             return null;
+        if (WispEngine.transparentWispSwitch()) {
+            return asyncImpl.getInetAddress();
+        }
         try {
             return getImpl().getInetAddress();
         } catch (SocketException e) {
@@ -740,6 +804,9 @@ public InetAddress getLocalAddress() {
         // This is for backward compatibility
         if (!isBound())
             return InetAddress.anyLocalAddress();
+        if (WispEngine.transparentWispSwitch()) {
+            return asyncImpl.getLocalAddress();
+        }
         InetAddress in = null;
         try {
             in = (InetAddress) getImpl().getOption(SocketOptions.SO_BINDADDR);
@@ -771,6 +838,9 @@ public InetAddress getLocalAddress() {
     public int getPort() {
         if (!isConnected())
             return 0;
+        if (WispEngine.transparentWispSwitch()) {
+            return asyncImpl.getPort();
+        }
         try {
             return getImpl().getPort();
         } catch (SocketException e) {
@@ -792,6 +862,9 @@ public int getPort() {
     public int getLocalPort() {
         if (!isBound())
             return -1;
+        if (WispEngine.transparentWispSwitch()) {
+            return asyncImpl.getLocalPort();
+        }
         try {
             return getImpl().getLocalPort();
         } catch(SocketException e) {
@@ -875,7 +948,7 @@ public SocketAddress getLocalSocketAddress() {
      * @since 1.4
      */
     public SocketChannel getChannel() {
-        return null;
+        return WispEngine.transparentWispSwitch() ? asyncImpl.getChannel() : null;
     }
 
     /**
@@ -929,6 +1002,10 @@ public InputStream getInputStream() throws IOException {
             throw new SocketException("Socket is not connected");
         if (isInputShutdown())
             throw new SocketException("Socket input is shutdown");
+
+        if (WispEngine.transparentWispSwitch()) {
+            return asyncImpl.getInputStream();
+        }
         InputStream in = this.in;
         if (in == null) {
             // wrap the input stream so that the close method closes this socket
@@ -1000,6 +1077,10 @@ public OutputStream getOutputStream() throws IOException {
             throw new SocketException("Socket is not connected");
         if (isOutputShutdown())
             throw new SocketException("Socket output is shutdown");
+
+        if (WispEngine.transparentWispSwitch()) {
+            return asyncImpl.getOutputStream();
+        }
         OutputStream out = this.out;
         if (out == null) {
             // wrap the output stream so that the close method closes this socket
@@ -1058,6 +1139,10 @@ public void close() throws IOException {
     public void setTcpNoDelay(boolean on) throws SocketException {
         if (isClosed())
             throw new SocketException("Socket is closed");
+        if (WispEngine.transparentWispSwitch()) {
+            asyncImpl.setTcpNoDelay(on);
+            return;
+        }
         getImpl().setOption(SocketOptions.TCP_NODELAY, Boolean.valueOf(on));
     }
 
@@ -1074,6 +1159,10 @@ public void setTcpNoDelay(boolean on) throws SocketException {
     public boolean getTcpNoDelay() throws SocketException {
         if (isClosed())
             throw new SocketException("Socket is closed");
+
+        if (WispEngine.transparentWispSwitch()) {
+            return asyncImpl.getTcpNoDelay();
+        }
         return ((Boolean) getImpl().getOption(SocketOptions.TCP_NODELAY)).booleanValue();
     }
 
@@ -1095,6 +1184,11 @@ public boolean getTcpNoDelay() throws SocketException {
     public void setSoLinger(boolean on, int linger) throws SocketException {
         if (isClosed())
             throw new SocketException("Socket is closed");
+
+        if (WispEngine.transparentWispSwitch()) {
+            asyncImpl.setSoLinger(on, linger);
+            return;
+        }
         if (!on) {
             getImpl().setOption(SocketOptions.SO_LINGER, on);
         } else {
@@ -1123,6 +1217,10 @@ public void setSoLinger(boolean on, int linger) throws SocketException {
     public int getSoLinger() throws SocketException {
         if (isClosed())
             throw new SocketException("Socket is closed");
+
+        if (WispEngine.transparentWispSwitch()) {
+            return asyncImpl.getSoLinger();
+        }
         Object o = getImpl().getOption(SocketOptions.SO_LINGER);
         if (o instanceof Integer) {
             return ((Integer) o).intValue();
@@ -1142,6 +1240,10 @@ public int getSoLinger() throws SocketException {
      * @since 1.4
      */
     public void sendUrgentData (int data) throws IOException  {
+        if (WispEngine.transparentWispSwitch()) {
+            asyncImpl.sendUrgentData(data);
+            return;
+        }
         if (!getImpl().supportsUrgentData ()) {
             throw new SocketException ("Urgent data not supported");
         }
@@ -1176,6 +1278,11 @@ public void sendUrgentData (int data) throws IOException  {
     public void setOOBInline(boolean on) throws SocketException {
         if (isClosed())
             throw new SocketException("Socket is closed");
+
+        if (WispEngine.transparentWispSwitch()) {
+            asyncImpl.setOOBInline(on);
+            return;
+        }
         getImpl().setOption(SocketOptions.SO_OOBINLINE, Boolean.valueOf(on));
     }
 
@@ -1193,6 +1300,10 @@ public void setOOBInline(boolean on) throws SocketException {
     public boolean getOOBInline() throws SocketException {
         if (isClosed())
             throw new SocketException("Socket is closed");
+
+        if (WispEngine.transparentWispSwitch()) {
+            return asyncImpl.getOOBInline();
+        }
         return ((Boolean) getImpl().getOption(SocketOptions.SO_OOBINLINE)).booleanValue();
     }
 
@@ -1219,6 +1330,10 @@ public synchronized void setSoTimeout(int timeout) throws SocketException {
         if (timeout < 0)
           throw new IllegalArgumentException("timeout can't be negative");
 
+        if (WispEngine.transparentWispSwitch()) {
+            asyncImpl.setSoTimeout(timeout);
+            return;
+        }
         getImpl().setOption(SocketOptions.SO_TIMEOUT, timeout);
     }
 
@@ -1236,6 +1351,10 @@ public synchronized void setSoTimeout(int timeout) throws SocketException {
     public synchronized int getSoTimeout() throws SocketException {
         if (isClosed())
             throw new SocketException("Socket is closed");
+
+        if (WispEngine.transparentWispSwitch()) {
+            return asyncImpl.getSoTimeout();
+        }
         Object o = getImpl().getOption(SocketOptions.SO_TIMEOUT);
         /* extra type safety */
         if (o instanceof Integer) {
@@ -1275,6 +1394,11 @@ public synchronized void setSendBufferSize(int size)
         }
         if (isClosed())
             throw new SocketException("Socket is closed");
+
+        if (WispEngine.transparentWispSwitch()) {
+            asyncImpl.setSendBufferSize(size);
+            return;
+        }
         getImpl().setOption(SocketOptions.SO_SNDBUF, size);
     }
 
@@ -1294,6 +1418,10 @@ public synchronized void setSendBufferSize(int size)
     public synchronized int getSendBufferSize() throws SocketException {
         if (isClosed())
             throw new SocketException("Socket is closed");
+
+        if (WispEngine.transparentWispSwitch()) {
+            return asyncImpl.getSendBufferSize();
+        }
         int result = 0;
         Object o = getImpl().getOption(SocketOptions.SO_SNDBUF);
         if (o instanceof Integer) {
@@ -1349,6 +1477,10 @@ public synchronized void setReceiveBufferSize(int size)
         }
         if (isClosed())
             throw new SocketException("Socket is closed");
+        if (WispEngine.transparentWispSwitch()) {
+            asyncImpl.setReceiveBufferSize(size);
+            return;
+        }
         getImpl().setOption(SocketOptions.SO_RCVBUF, size);
     }
 
@@ -1368,6 +1500,9 @@ public synchronized int getReceiveBufferSize()
     throws SocketException{
         if (isClosed())
             throw new SocketException("Socket is closed");
+        if (WispEngine.transparentWispSwitch()) {
+            return asyncImpl.getReceiveBufferSize();
+        }
         int result = 0;
         Object o = getImpl().getOption(SocketOptions.SO_RCVBUF);
         if (o instanceof Integer) {
@@ -1388,6 +1523,10 @@ public synchronized int getReceiveBufferSize()
     public void setKeepAlive(boolean on) throws SocketException {
         if (isClosed())
             throw new SocketException("Socket is closed");
+        if (WispEngine.transparentWispSwitch()) {
+            asyncImpl.setKeepAlive(on);
+            return;
+        }
         getImpl().setOption(SocketOptions.SO_KEEPALIVE, Boolean.valueOf(on));
     }
 
@@ -1404,6 +1543,9 @@ public void setKeepAlive(boolean on) throws SocketException {
     public boolean getKeepAlive() throws SocketException {
         if (isClosed())
             throw new SocketException("Socket is closed");
+        if (WispEngine.transparentWispSwitch()) {
+            return asyncImpl.getKeepAlive();
+        }
         return ((Boolean) getImpl().getOption(SocketOptions.SO_KEEPALIVE)).booleanValue();
     }
 
@@ -1459,6 +1601,10 @@ public void setTrafficClass(int tc) throws SocketException {
 
         if (isClosed())
             throw new SocketException("Socket is closed");
+        if (WispEngine.transparentWispSwitch()) {
+            asyncImpl.setTrafficClass(tc);
+            return;
+        }
         try {
             getImpl().setOption(SocketOptions.IP_TOS, tc);
         } catch (SocketException se) {
@@ -1486,6 +1632,9 @@ public void setTrafficClass(int tc) throws SocketException {
      * @see SocketOptions#IP_TOS
      */
     public int getTrafficClass() throws SocketException {
+        if (WispEngine.transparentWispSwitch()) {
+            return asyncImpl.getTrafficClass();
+        }
         return ((Integer) (getImpl().getOption(SocketOptions.IP_TOS))).intValue();
     }
 
@@ -1527,6 +1676,10 @@ public int getTrafficClass() throws SocketException {
     public void setReuseAddress(boolean on) throws SocketException {
         if (isClosed())
             throw new SocketException("Socket is closed");
+        if (WispEngine.transparentWispSwitch()) {
+            asyncImpl.setReuseAddress(on);
+            return;
+        }
         getImpl().setOption(SocketOptions.SO_REUSEADDR, Boolean.valueOf(on));
     }
 
@@ -1541,6 +1694,9 @@ public void setReuseAddress(boolean on) throws SocketException {
      * @see #setReuseAddress(boolean)
      */
     public boolean getReuseAddress() throws SocketException {
+        if (WispEngine.transparentWispSwitch()) {
+            return asyncImpl.getReuseAddress();
+        }
         if (isClosed())
             throw new SocketException("Socket is closed");
         return ((Boolean) (getImpl().getOption(SocketOptions.SO_REUSEADDR))).booleanValue();
@@ -1568,6 +1724,10 @@ public boolean getReuseAddress() throws SocketException {
      * @see #isClosed
      */
     public synchronized void close() throws IOException {
+        if (WispEngine.transparentWispSwitch()) {
+            asyncImpl.close();
+            return;
+        }
         synchronized(closeLock) {
             if (isClosed())
                 return;
@@ -1603,6 +1763,10 @@ public void shutdownInput() throws IOException
             throw new SocketException("Socket is not connected");
         if (isInputShutdown())
             throw new SocketException("Socket input is already shutdown");
+        if (WispEngine.transparentWispSwitch()) {
+            asyncImpl.shutdownInput();
+            return;
+        }
         getImpl().shutdownInput();
         shutIn = true;
     }
@@ -1633,6 +1797,10 @@ public void shutdownOutput() throws IOException
             throw new SocketException("Socket is not connected");
         if (isOutputShutdown())
             throw new SocketException("Socket output is already shutdown");
+        if (WispEngine.transparentWispSwitch()) {
+            asyncImpl.shutdownOutput();
+            return;
+        }
         getImpl().shutdownOutput();
         shutOut = true;
     }
@@ -1643,6 +1811,9 @@ public void shutdownOutput() throws IOException
      * @return  a string representation of this socket.
      */
     public String toString() {
+        if (WispEngine.transparentWispSwitch()) {
+            return asyncImpl.toString();
+        }
         try {
             if (isConnected())
                 return "Socket[addr=" + getImpl().getInetAddress() +
@@ -1665,6 +1836,9 @@ public String toString() {
      * @since 1.4
      */
     public boolean isConnected() {
+        if (WispEngine.transparentWispSwitch()) {
+            return asyncImpl.isConnected();
+        }
         return connected;
     }
 
@@ -1681,6 +1855,9 @@ public boolean isConnected() {
      * @see #bind
      */
     public boolean isBound() {
+        if (WispEngine.transparentWispSwitch()) {
+            return asyncImpl.isBound();
+        }
         return bound;
     }
 
@@ -1692,6 +1869,9 @@ public boolean isBound() {
      * @see #close
      */
     public boolean isClosed() {
+        if (WispEngine.transparentWispSwitch()) {
+            return asyncImpl.isClosed();
+        }
         synchronized(closeLock) {
             return closed;
         }
@@ -1705,6 +1885,9 @@ public boolean isClosed() {
      * @see #shutdownInput
      */
     public boolean isInputShutdown() {
+        if (WispEngine.transparentWispSwitch()) {
+            return asyncImpl.isInputShutdown();
+        }
         return shutIn;
     }
 
@@ -1716,6 +1899,9 @@ public boolean isInputShutdown() {
      * @see #shutdownOutput
      */
     public boolean isOutputShutdown() {
+        if (WispEngine.transparentWispSwitch()) {
+            return asyncImpl.isOutputShutdown();
+        }
         return shutOut;
     }
 
@@ -1765,6 +1951,10 @@ static SocketImplFactory socketImplFactory() {
     public static synchronized void setSocketImplFactory(SocketImplFactory fac)
         throws IOException
     {
+        if (WispEngine.transparentWispSwitch()) {
+            throw new UnsupportedOperationException();
+        }
+
         if (factory != null) {
             throw new SocketException("factory already defined");
         }
@@ -1853,7 +2043,11 @@ public  Socket setOption(SocketOption name, T value) throws IOException {
         Objects.requireNonNull(name);
         if (isClosed())
             throw new SocketException("Socket is closed");
-        getImpl().setOption(name, value);
+        if (WispEngine.transparentWispSwitch()) {
+            getChannel().setOption(name, value);
+        } else {
+            getImpl().setOption(name, value);
+        }
         return this;
     }
 
@@ -1885,7 +2079,7 @@ public  T getOption(SocketOption name) throws IOException {
         Objects.requireNonNull(name);
         if (isClosed())
             throw new SocketException("Socket is closed");
-        return getImpl().getOption(name);
+        return WispEngine.transparentWispSwitch() ? getChannel().getOption(name) : getImpl().getOption(name);
     }
 
     // cache of unmodifiable impl options. Possibly set racy, in impl we trust
diff --git a/src/java.base/share/classes/java/util/concurrent/LinkedBlockingDeque.java b/src/java.base/share/classes/java/util/concurrent/LinkedBlockingDeque.java
index 926b4b8cfb6..3543a9b8c54 100644
--- a/src/java.base/share/classes/java/util/concurrent/LinkedBlockingDeque.java
+++ b/src/java.base/share/classes/java/util/concurrent/LinkedBlockingDeque.java
@@ -200,6 +200,9 @@ public LinkedBlockingDeque(Collection c) {
         addAll(c);
     }
 
+    int getCapacity() {
+        return capacity;
+    }
 
     // Basic linking and unlinking operations, called only while holding lock
 
diff --git a/src/java.base/share/classes/java/util/concurrent/LinkedBlockingQueue.java b/src/java.base/share/classes/java/util/concurrent/LinkedBlockingQueue.java
index a67be9a987d..8487ac2092a 100644
--- a/src/java.base/share/classes/java/util/concurrent/LinkedBlockingQueue.java
+++ b/src/java.base/share/classes/java/util/concurrent/LinkedBlockingQueue.java
@@ -166,6 +166,10 @@ static class Node {
     @SuppressWarnings("serial") // Classes implementing Condition may be serializable.
     private final Condition notFull = putLock.newCondition();
 
+    int getCapacity() {
+        return capacity;
+    }
+
     /**
      * Signals a waiting take. Called only from put/offer (which do not
      * otherwise ordinarily lock takeLock.)
diff --git a/src/java.base/share/classes/java/util/concurrent/ScheduledThreadPoolExecutor.java b/src/java.base/share/classes/java/util/concurrent/ScheduledThreadPoolExecutor.java
index 87e2ea49205..cc9448ffe38 100644
--- a/src/java.base/share/classes/java/util/concurrent/ScheduledThreadPoolExecutor.java
+++ b/src/java.base/share/classes/java/util/concurrent/ScheduledThreadPoolExecutor.java
@@ -35,6 +35,8 @@
 
 package java.util.concurrent;
 
+import com.alibaba.wisp.engine.WispEngine;
+
 import static java.util.concurrent.TimeUnit.MILLISECONDS;
 import static java.util.concurrent.TimeUnit.NANOSECONDS;
 
@@ -339,6 +341,12 @@ private void delayedExecute(RunnableScheduledFuture task) {
         if (isShutdown())
             reject(task);
         else {
+            if (WispEngine.transparentWispSwitch() &&
+                    WispEngine.enableThreadAsWisp() &&
+                    putToCurrentEngine() && !task.isPeriodic() && task.getDelay(TimeUnit.MICROSECONDS) <= 0) {
+                WispEngine.dispatch(task);
+                return;
+            }
             super.getQueue().add(task);
             if (!canRunInCurrentRunState(task) && remove(task))
                 task.cancel(false);
diff --git a/src/java.base/share/classes/java/util/concurrent/SynchronousQueue.java b/src/java.base/share/classes/java/util/concurrent/SynchronousQueue.java
index 926897c4f1d..19a34dfe899 100644
--- a/src/java.base/share/classes/java/util/concurrent/SynchronousQueue.java
+++ b/src/java.base/share/classes/java/util/concurrent/SynchronousQueue.java
@@ -36,6 +36,10 @@
 
 package java.util.concurrent;
 
+import com.alibaba.wisp.engine.WispEngine;
+import jdk.internal.access.SharedSecrets;
+import jdk.internal.access.WispEngineAccess;
+
 import java.lang.invoke.MethodHandles;
 import java.lang.invoke.VarHandle;
 import java.util.AbstractQueue;
@@ -91,6 +95,8 @@ public class SynchronousQueue extends AbstractQueue
     implements BlockingQueue, java.io.Serializable {
     private static final long serialVersionUID = -3223113410248163686L;
 
+    private static WispEngineAccess WEA = SharedSecrets.getWispEngineAccess();
+
     /*
      * This class implements extensions of the dual stack and dual
      * queue algorithms described in "Nonblocking Concurrent Objects
@@ -397,7 +403,8 @@ E transfer(E e, boolean timed, long nanos) {
                                     ForkJoinPool.managedBlock(s);
                                 } catch (InterruptedException cannotHappen) { }
                                 LockSupport.setCurrentBlocker(null);
-                            } else if (nanos > SPIN_FOR_TIMEOUT_THRESHOLD)
+                            } else if (nanos > SPIN_FOR_TIMEOUT_THRESHOLD ||
+                                      WispEngine.transparentWispSwitch() && WEA.hasMoreTasks())
                                 LockSupport.parkNanos(this, nanos);
                         }
                         if (stat == 1)
@@ -700,7 +707,8 @@ else if (t.casNext(null, (s != null) ? s :
                                 } catch (InterruptedException cannotHappen) { }
                                 LockSupport.setCurrentBlocker(null);
                             }
-                            else if (nanos > SPIN_FOR_TIMEOUT_THRESHOLD)
+                            else if (nanos > SPIN_FOR_TIMEOUT_THRESHOLD ||
+                                    WispEngine.transparentWispSwitch() && WEA.hasMoreTasks())
                                 LockSupport.parkNanos(this, nanos);
                         }
                         if (stat == 1)
diff --git a/src/java.base/share/classes/java/util/concurrent/ThreadPoolExecutor.java b/src/java.base/share/classes/java/util/concurrent/ThreadPoolExecutor.java
index 49d2a29b262..24c74bf6744 100644
--- a/src/java.base/share/classes/java/util/concurrent/ThreadPoolExecutor.java
+++ b/src/java.base/share/classes/java/util/concurrent/ThreadPoolExecutor.java
@@ -35,6 +35,10 @@
 
 package java.util.concurrent;
 
+import com.alibaba.wisp.engine.WispEngine;
+import com.alibaba.wisp.engine.WispWorkerContainer;
+import jdk.internal.access.SharedSecrets;
+
 import java.util.ArrayList;
 import java.util.ConcurrentModificationException;
 import java.util.HashSet;
@@ -1309,6 +1313,108 @@ public ThreadPoolExecutor(int corePoolSize,
         this.keepAliveTime = unit.toNanos(keepAliveTime);
         this.threadFactory = threadFactory;
         this.handler = handler;
+        if (WispEngine.transparentWispSwitch() && WispEngine.enableThreadAsWisp()) {
+            this.wispTaskLimit = estimateWispTaskLimit();
+        }
+    }
+
+    private Boolean putToLocalEngine;
+
+    boolean putToCurrentEngine() {
+        if (putToLocalEngine == null)
+            putToLocalEngine = SharedSecrets.getWispEngineAccess().ifPutToCurrentEngine();
+        return putToLocalEngine;
+    }
+
+    private Boolean putToManagedThread;
+
+    private boolean ifPutToManagedThread() {
+        if (putToManagedThread == null)
+            putToManagedThread = SharedSecrets.getWispEngineAccess().ifPutToManagedThread();
+        return putToManagedThread;
+    }
+
+    private String threadNamePrefix;
+    private AtomicInteger nextThreadId = new AtomicInteger();
+    private String getNextThreadName() {
+        if (threadNamePrefix == null) {
+            if (getThreadFactory() == null) {
+                threadNamePrefix = "tp-coroutine";
+            } else {
+                threadNamePrefix = getThreadFactory().newThread(() -> {}).getName().replaceAll("-?\\d+$", "");
+            }
+        }
+        return threadNamePrefix + "-" + (nextThreadId.getAndIncrement() & 0xFFFFFFF) % getMaximumPoolSize();
+    }
+
+    /**
+     * running wisp task counter for this {@code ThreadPoolExecutor}
+     */
+    private final AtomicInteger runningWispTaskCount = new AtomicInteger(0);
+
+    private static int accumulate(int v1, int v2) {
+        assert v1 > 0 && v2 > 0;
+        if (v1 >= Integer.MAX_VALUE - v2) {
+            return Integer.MAX_VALUE;
+        }
+        return v1 + v2;
+    }
+
+    private int wispTaskLimit = Integer.MAX_VALUE;
+
+    private int estimateLinkedBlockingDequeLimit(LinkedBlockingDeque queue) {
+        int capacity = queue.getCapacity();
+        return accumulate(capacity, getMaximumPoolSize());
+    }
+
+    private int estimateLinkedBlockingQueueLimit(LinkedBlockingQueue queue) {
+        int capacity = queue.getCapacity();
+        return accumulate(capacity, getMaximumPoolSize());
+    }
+
+    private int estimateArrayBlockingQueueLimit(ArrayBlockingQueue queue) {
+        return accumulate(queue.items.length, getMaximumPoolSize());
+    }
+
+    /**
+     * estimate wisp task limit for this {@code ThreadPoolExecutor} according to {@code ThreadPoolExecutor#workQueue}
+     * @return estimated wisp task limit
+     */
+    private int estimateWispTaskLimit() {
+        if (workQueue instanceof LinkedBlockingDeque) {
+            return estimateLinkedBlockingDequeLimit((LinkedBlockingDeque) workQueue);
+        } else if (workQueue instanceof LinkedBlockingQueue) {
+            return estimateLinkedBlockingQueueLimit((LinkedBlockingQueue) workQueue);
+        } else if (workQueue instanceof ArrayBlockingQueue) {
+            return estimateArrayBlockingQueueLimit((ArrayBlockingQueue) workQueue);
+        } else if (workQueue instanceof SynchronousQueue ) {
+            return getMaximumPoolSize();
+        } else if (workQueue instanceof DelayQueue || workQueue instanceof PriorityBlockingQueue
+                || workQueue instanceof LinkedTransferQueue) {
+            return Integer.MAX_VALUE;
+        }
+        // default value
+        return Integer.MAX_VALUE;
+    }
+
+    private long getWispTaskLimit() {
+        return wispTaskLimit;
+    }
+
+    /**
+     * Increment running wisp task counter if this {@code ThreadPoolExecutor} is not full.
+     * @return true if successful.
+     */
+    private boolean incrementWispTaskCount() {
+        for (;;) {
+            int cur = runningWispTaskCount.get();
+            if (cur >= getWispTaskLimit()) {
+                return false;
+            }
+            if (runningWispTaskCount.compareAndSet(cur, cur + 1)) {
+                return true;
+            }
+        }
     }
 
     /**
@@ -1328,6 +1434,36 @@ public ThreadPoolExecutor(int corePoolSize,
     public void execute(Runnable command) {
         if (command == null)
             throw new NullPointerException();
+
+        if (WispEngine.transparentWispSwitch() && WispEngine.enableThreadAsWisp()) {
+            Runnable wrappedCommand = command;
+            if (SharedSecrets.getWispEngineAccess().useThreadPoolLimit()
+                    && (putToCurrentEngine() || ifPutToManagedThread())) {
+                if (!incrementWispTaskCount()) {
+                    reject(command);
+                    return;
+                }
+                // wrap origin task
+                wrappedCommand = () -> {
+                    try {
+                        command.run();
+                    } finally {
+                        int restTasks = runningWispTaskCount.decrementAndGet();
+                        assert restTasks >= 0;
+                    }
+                };
+
+            }
+            if (putToCurrentEngine()) {
+                SharedSecrets.getWispEngineAccess().dispatch(wrappedCommand, getNextThreadName());
+                return;
+            } else if (ifPutToManagedThread()) {
+                String usage = SharedSecrets.getWispEngineAccess().getThreadUsage(
+                        SharedSecrets.getJavaLangAccess().currentThread0().getName());
+                WispWorkerContainer.INSTANCE.dispatch(usage, getNextThreadName(), wrappedCommand, null);
+                return;
+            }
+        }
         /*
          * Proceed in 3 steps:
          *
diff --git a/src/java.base/share/classes/java/util/concurrent/locks/AbstractQueuedSynchronizer.java b/src/java.base/share/classes/java/util/concurrent/locks/AbstractQueuedSynchronizer.java
index 921150f58ca..159e9e29dcd 100644
--- a/src/java.base/share/classes/java/util/concurrent/locks/AbstractQueuedSynchronizer.java
+++ b/src/java.base/share/classes/java/util/concurrent/locks/AbstractQueuedSynchronizer.java
@@ -35,6 +35,10 @@
 
 package java.util.concurrent.locks;
 
+import com.alibaba.wisp.engine.WispEngine;
+import jdk.internal.access.SharedSecrets;
+import jdk.internal.access.WispEngineAccess;
+
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Date;
@@ -307,6 +311,8 @@ public abstract class AbstractQueuedSynchronizer
 
     private static final long serialVersionUID = 7373984972572414691L;
 
+    private static WispEngineAccess WEA = SharedSecrets.getWispEngineAccess();
+
     /**
      * Creates a new {@code AbstractQueuedSynchronizer} instance
      * with initial synchronization state of zero.
@@ -713,7 +719,8 @@ else if (!casTail(t, node))
                 spins = postSpins = (byte)((postSpins << 1) | 1);
                 if (!timed)
                     LockSupport.park(this);
-                else if ((nanos = time - System.nanoTime()) > 0L)
+                else if ((nanos = time - System.nanoTime()) > 0L ||
+                        WispEngine.transparentWispSwitch() && WEA.hasMoreTasks())
                     LockSupport.parkNanos(this, nanos);
                 else
                     break;
diff --git a/src/java.base/share/classes/jdk/internal/access/JavaLangAccess.java b/src/java.base/share/classes/jdk/internal/access/JavaLangAccess.java
index b68490ad7a3..9bde06e7382 100644
--- a/src/java.base/share/classes/jdk/internal/access/JavaLangAccess.java
+++ b/src/java.base/share/classes/jdk/internal/access/JavaLangAccess.java
@@ -43,6 +43,8 @@
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.stream.Stream;
 
+import com.alibaba.wisp.engine.WispEngine;
+import com.alibaba.wisp.engine.WispTask;
 import jdk.internal.module.ServicesCatalog;
 import jdk.internal.reflect.ConstantPool;
 import sun.reflect.annotation.AnnotationType;
@@ -410,4 +412,27 @@ public interface JavaLangAccess {
      * @param statusCode the status code
      */
     void exit(int statusCode);
+
+    /**
+     * Returns a reference to the currently executing thread object.
+     */
+    Thread currentThread0();
+
+    void yield0();
+
+    void setWispEngine(Thread thread, WispEngine engine);
+
+    WispEngine getWispEngine(Thread thread);
+
+    void setWispTask(Thread thread, WispTask task);
+
+    WispTask getWispTask(Thread thread);
+
+    void setWispAlive(Thread thread, boolean b);
+
+    boolean isInNative(Thread thread);
+
+    void threadExit(Thread thread);
+
+    void wispBooted();
 }
diff --git a/src/java.base/share/classes/jdk/internal/access/SharedSecrets.java b/src/java.base/share/classes/jdk/internal/access/SharedSecrets.java
index ea28bb8747e..60e33edc288 100644
--- a/src/java.base/share/classes/jdk/internal/access/SharedSecrets.java
+++ b/src/java.base/share/classes/jdk/internal/access/SharedSecrets.java
@@ -25,6 +25,9 @@
 
 package jdk.internal.access;
 
+import com.alibaba.wisp.engine.WispEngine;
+import sun.nio.ch.EpollAccess;
+
 import javax.crypto.SealedObject;
 import javax.crypto.spec.SecretKeySpec;
 import java.io.ObjectInputFilter;
@@ -41,6 +44,7 @@
 import java.io.RandomAccessFile;
 import java.security.ProtectionDomain;
 import java.security.Signature;
+import jdk.internal.misc.Unsafe;
 
 /** A repository of "shared secrets", which are a mechanism for
     calling implementation-private methods in another package without
@@ -83,6 +87,8 @@ public class SharedSecrets {
     private static JavaSecuritySpecAccess javaSecuritySpecAccess;
     private static JavaxCryptoSealedObjectAccess javaxCryptoSealedObjectAccess;
     private static JavaxCryptoSpecAccess javaxCryptoSpecAccess;
+    private static WispEngineAccess wispEngineAccess;
+    private static EpollAccess epollAccess;
 
     public static void setJavaUtilCollectionAccess(JavaUtilCollectionAccess juca) {
         javaUtilCollectionAccess = juca;
@@ -457,4 +463,27 @@ private static void ensureClassInitialized(Class c) {
             MethodHandles.lookup().ensureInitialized(c);
         } catch (IllegalAccessException e) {}
     }
+
+    public static WispEngineAccess getWispEngineAccess() {
+        return wispEngineAccess;
+    }
+
+    public static void setWispEngineAccess(WispEngineAccess wispEngineAccess) {
+        SharedSecrets.wispEngineAccess = wispEngineAccess;
+    }
+
+    public static UnsafeAccess getUnsafeAccess() {
+        return Unsafe.access;
+    }
+
+    public static void setEpollAccess(EpollAccess ea) {
+        epollAccess = ea;
+    }
+
+    public static EpollAccess getEpollAccess() {
+        if (epollAccess == null) {
+            EpollAccess.initializeEpoll();
+        }
+        return epollAccess;
+    }
 }
diff --git a/src/java.base/share/classes/jdk/internal/access/UnsafeAccess.java b/src/java.base/share/classes/jdk/internal/access/UnsafeAccess.java
new file mode 100644
index 00000000000..741c2b642f6
--- /dev/null
+++ b/src/java.base/share/classes/jdk/internal/access/UnsafeAccess.java
@@ -0,0 +1,7 @@
+package jdk.internal.access;
+
+public interface UnsafeAccess {
+    void unpark0(Object thread);
+
+    void park0(boolean isAbsolute, long time);
+}
diff --git a/src/java.base/share/classes/jdk/internal/access/WispEngineAccess.java b/src/java.base/share/classes/jdk/internal/access/WispEngineAccess.java
new file mode 100644
index 00000000000..ecc7d5f8e96
--- /dev/null
+++ b/src/java.base/share/classes/jdk/internal/access/WispEngineAccess.java
@@ -0,0 +1,76 @@
+package jdk.internal.access;
+
+import com.alibaba.wisp.engine.WispTask;
+
+import java.io.IOException;
+import java.nio.channels.SelectableChannel;
+
+public interface WispEngineAccess {
+
+    WispTask getCurrentTask();
+
+    void dispatch(Runnable runnable, String name);
+
+    void eventLoop();
+
+    void registerEvent(SelectableChannel ch, int events) throws IOException;
+
+    void registerEpollEvent(int epFd) throws IOException;
+
+    void unregisterEvent();
+
+    void yieldOnBlocking();
+
+    void addTimer(long deadlineNano);
+
+    void cancelTimer();
+
+    void sleep(long ms);
+
+    void yield();
+
+    boolean isThreadTask(WispTask task);
+
+    boolean isTimeout();
+
+    void park(long timeoutMs);
+
+    void unpark(WispTask task);
+
+    void destroy();
+
+    boolean isAlive(WispTask task);
+
+    void interrupt(WispTask task);
+
+    boolean isInterrupted(WispTask task);
+
+    boolean testInterruptedAndClear(WispTask task, boolean clear);
+
+     T runInCritical(CheckedSupplier supplier);
+
+    @FunctionalInterface
+    interface CheckedSupplier {
+        T get() throws Throwable;
+    }
+
+    boolean hasMoreTasks();
+
+    boolean runningAsCoroutine(Thread t);
+
+    boolean usingWispEpoll(Thread t);
+
+    boolean ifPutToCurrentEngine();
+
+    boolean ifProxySelector();
+
+    boolean ifSpinSelector();
+
+    boolean ifPutToManagedThread();
+
+    boolean useThreadPoolLimit();
+
+    String getThreadUsage(String threadName);
+
+    boolean tryStartThreadAsWisp(Thread thread, Runnable target);
+}
diff --git a/src/java.base/share/classes/jdk/internal/misc/Unsafe.java b/src/java.base/share/classes/jdk/internal/misc/Unsafe.java
index 22aa09c9d62..758ab33d935 100644
--- a/src/java.base/share/classes/jdk/internal/misc/Unsafe.java
+++ b/src/java.base/share/classes/jdk/internal/misc/Unsafe.java
@@ -25,6 +25,9 @@
 
 package jdk.internal.misc;
 
+import com.alibaba.wisp.engine.WispEngine;
+import jdk.internal.access.SharedSecrets;
+import jdk.internal.access.UnsafeAccess;
 import jdk.internal.ref.Cleaner;
 import jdk.internal.vm.annotation.ForceInline;
 import jdk.internal.vm.annotation.IntrinsicCandidate;
@@ -64,6 +67,19 @@ private Unsafe() {}
 
     private static final Unsafe theUnsafe = new Unsafe();
 
+    public static final UnsafeAccess access = new UnsafeAccess() {
+    // static final UnsafeAccess access = new UnsafeAccess() {
+        @Override
+        public void unpark0(Object thread) {
+            theUnsafe.unpark0(thread);
+        }
+
+        @Override
+        public void park0(boolean isAbsolute, long time) {
+            theUnsafe.park0(isAbsolute, time);
+        }
+    };
+
     /**
      * Provides the caller with the capability of performing unsafe
      * operations.
@@ -2382,8 +2398,23 @@ public final void putDoubleOpaque(Object o, long offset, double x) {
      *
      * @param thread the thread to unpark.
      */
+    public void unpark(Object thread) {
+        if (WispEngine.transparentWispSwitch()) {
+            if (thread instanceof Thread) {
+                SharedSecrets.getWispEngineAccess().unpark(
+                        SharedSecrets.getJavaLangAccess().getWispTask((Thread) thread));
+            }
+            return;
+        }
+        unpark0(thread);
+    }
+
+    /**
+     * Unblock the given thread. Always use the thread semantic.
+     * @param thread
+     */
     @IntrinsicCandidate
-    public native void unpark(Object thread);
+    private native void unpark0(Object thread);
 
     /**
      * Blocks current thread, returning when a balancing
@@ -2396,8 +2427,26 @@ public final void putDoubleOpaque(Object o, long offset, double x) {
      * because {@code unpark} is, so it would be strange to place it
      * elsewhere.
      */
+    public void park(boolean isAbsolute, long time) {
+        if (WispEngine.transparentWispSwitch()) {
+            if (time <= 0) { // non-timeouted park
+                SharedSecrets.getWispEngineAccess().park(0);
+            }
+            long timeout = !isAbsolute ? time:
+                    java.util.concurrent.TimeUnit.MILLISECONDS.toNanos(time - System.currentTimeMillis());
+            if (timeout > 0) {
+                SharedSecrets.getWispEngineAccess().park(timeout);
+            }
+            return;
+        }
+        park0(isAbsolute, time);
+    }
+
+    /**
+     * Block current thread. Always use the thread semantic.
+     */
     @IntrinsicCandidate
-    public native void park(boolean isAbsolute, long time);
+    private native void park0(boolean isAbsolute, long time);
 
     /**
      * Gets the load average in the system run queue assigned
diff --git a/src/java.base/share/classes/module-info.java b/src/java.base/share/classes/module-info.java
index 9acb56bb0c7..a2d8bbf5624 100644
--- a/src/java.base/share/classes/module-info.java
+++ b/src/java.base/share/classes/module-info.java
@@ -130,6 +130,7 @@
     exports javax.security.auth.spi;
     exports javax.security.auth.x500;
     exports javax.security.cert;
+    exports com.alibaba.wisp.engine;
 
 
     // additional qualified exports may be inserted at build time
diff --git a/src/java.base/share/classes/sun/nio/ch/EpollAccess.java b/src/java.base/share/classes/sun/nio/ch/EpollAccess.java
new file mode 100644
index 00000000000..7be6a227bc2
--- /dev/null
+++ b/src/java.base/share/classes/sun/nio/ch/EpollAccess.java
@@ -0,0 +1,35 @@
+package sun.nio.ch;
+
+import jdk.internal.misc.Unsafe;
+
+import java.io.IOException;
+
+
+public interface EpollAccess {
+
+    int EPOLL_CTL_ADD = EPoll.EPOLL_CTL_ADD;
+    int EPOLL_CTL_DEL = EPoll.EPOLL_CTL_DEL;
+    int EPOLL_CTL_MOD = EPoll.EPOLL_CTL_MOD;
+    int EPOLLONESHOT = EPoll.EPOLLONESHOT;
+    int ENOENT = EPoll.ENOENT;
+
+    long allocatePollArray(int count);
+
+    void freePollArray(long address);
+
+    long getEvent(long address, int i);
+
+    int getDescriptor(long eventAddress);
+
+    int getEvents(long eventAddress);
+
+    int epollCreate() throws IOException;
+
+    int epollCtl(int epfd, int opcode, int fd, int events);
+
+    int epollWait(int epfd, long pollAddress, int numfds) throws IOException;
+
+    static void initializeEpoll() {
+        Unsafe.getUnsafe().ensureClassInitialized(EPoll.class);
+    }
+}
diff --git a/src/java.base/share/classes/sun/nio/ch/ServerSocketChannelImpl.java b/src/java.base/share/classes/sun/nio/ch/ServerSocketChannelImpl.java
index bb07894fb33..4aeadee8d45 100644
--- a/src/java.base/share/classes/sun/nio/ch/ServerSocketChannelImpl.java
+++ b/src/java.base/share/classes/sun/nio/ch/ServerSocketChannelImpl.java
@@ -55,6 +55,9 @@
 import static java.net.StandardProtocolFamily.INET6;
 import static java.net.StandardProtocolFamily.UNIX;
 
+import com.alibaba.wisp.engine.WispEngine;
+import jdk.internal.access.SharedSecrets;
+import jdk.internal.access.WispEngineAccess;
 import sun.net.NetHooks;
 import sun.net.ext.ExtendedSocketOptions;
 
@@ -66,6 +69,8 @@ class ServerSocketChannelImpl
     extends ServerSocketChannel
     implements SelChImpl
 {
+    private static final WispEngineAccess WEA = SharedSecrets.getWispEngineAccess();
+
     // Used to make native close and configure calls
     private static final NativeDispatcher nd = new SocketDispatcher();
 
@@ -386,16 +391,26 @@ public SocketChannel accept() throws IOException {
         acceptLock.lock();
         try {
             boolean blocking = isBlocking();
+            final boolean wispAndBlocking = WispEngine.transparentWispSwitch() && blocking;
             try {
                 begin(blocking);
                 n = implAccept(this.fd, newfd, saa);
                 if (blocking) {
                     while (IOStatus.okayToRetry(n) && isOpen()) {
-                        park(Net.POLLIN);
+                        if (wispAndBlocking && n < 0) {
+                            WEA.registerEvent(this, SelectionKey.OP_ACCEPT);
+                            WEA.park(-1);
+                        }
+                        else {
+                            park(Net.POLLIN);
+                        }
                         n = implAccept(this.fd, newfd, saa);
                     }
                 }
             } finally {
+                if (wispAndBlocking) {
+                    IOUtil.configureBlocking(fd, true);
+                }
                 end(blocking, n > 0);
                 assert IOStatus.check(n);
             }
diff --git a/src/java.base/share/classes/sun/nio/ch/WispServerSocketImpl.java b/src/java.base/share/classes/sun/nio/ch/WispServerSocketImpl.java
new file mode 100644
index 00000000000..b9b55a3d799
--- /dev/null
+++ b/src/java.base/share/classes/sun/nio/ch/WispServerSocketImpl.java
@@ -0,0 +1,166 @@
+package sun.nio.ch;
+
+import jdk.internal.access.SharedSecrets;
+import jdk.internal.access.WispEngineAccess;
+
+import java.io.IOException;
+import java.net.*;
+import java.nio.channels.SelectionKey;
+import java.nio.channels.ServerSocketChannel;
+import java.nio.channels.SocketChannel;
+import java.util.concurrent.TimeUnit;
+
+
+// Make a server socket channel be like a socket and yield on block
+
+public class WispServerSocketImpl
+{
+    private static WispEngineAccess WEA = SharedSecrets.getWispEngineAccess();
+    // The channel being adapted
+    private ServerSocketChannelImpl ssc = null;
+
+    // Timeout "option" value for accepts
+    private volatile int timeout = 0;
+
+    public WispServerSocketImpl() {
+    }
+
+    public void bind(SocketAddress local) throws IOException {
+        bind(local, 50);
+    }
+
+    public void bind(SocketAddress local, int backlog) throws IOException {
+        if (local == null)
+            local = new InetSocketAddress(0);
+        try {
+            getChannelImpl().bind(local, backlog);
+        } catch (Exception x) {
+            Net.translateException(x);
+        }
+    }
+
+    public InetAddress getInetAddress() {
+        if (ssc == null || !ssc.isBound())
+            return null;
+        return Net.getRevealedLocalAddress(ssc.localAddress()).getAddress();
+    }
+
+    public int getLocalPort() {
+        if (ssc == null || !ssc.isBound())
+            return -1;
+        return Net.asInetSocketAddress(ssc.localAddress()).getPort();
+    }
+
+    public Socket accept() throws IOException {
+
+        final ServerSocketChannel ch = getChannelImpl();
+        try {
+            SocketChannel res;
+
+            if (getSoTimeout() > 0) {
+                WEA.addTimer(System.nanoTime() + TimeUnit.MILLISECONDS.toNanos(getSoTimeout()));
+            }
+
+            while ((res = ch.accept()) == null) {
+                WEA.registerEvent(ch, SelectionKey.OP_ACCEPT);
+                WEA.park(-1);
+
+                if (getSoTimeout() > 0 && WEA.isTimeout()) {
+                    throw new SocketTimeoutException("time out");
+                }
+            }
+            res.configureBlocking(false);
+            return new Socket(res);
+
+        } catch (Exception x) {
+            Net.translateException(x, true);
+            return null;
+        } finally {
+            if (getSoTimeout() > 0) {
+                WEA.cancelTimer();
+            }
+            WEA.unregisterEvent();
+        }
+    }
+
+    public void close() throws IOException {
+        if (ssc != null) {
+            ssc.close();
+        }
+    }
+
+    public ServerSocketChannel getChannel() {
+        return ssc;
+    }
+
+    public boolean isBound() {
+        return ssc != null && ssc.isBound();
+    }
+
+    public boolean isClosed() {
+        return ssc != null && !ssc.isOpen();
+    }
+
+    public void setSoTimeout(int timeout) throws SocketException {
+        this.timeout = timeout;
+    }
+
+    public int getSoTimeout() throws IOException {
+        return timeout;
+    }
+
+    public void setReuseAddress(boolean on) throws SocketException {
+        try {
+            getChannelImpl().setOption(StandardSocketOptions.SO_REUSEADDR, on);
+        } catch (IOException x) {
+            Net.translateToSocketException(x);
+        }
+    }
+
+    public boolean getReuseAddress() throws SocketException {
+        try {
+            return getChannelImpl().getOption(StandardSocketOptions.SO_REUSEADDR);
+        } catch (IOException x) {
+            Net.translateToSocketException(x);
+            return false;       // Never happens
+        }
+    }
+
+    public String toString() {
+        if (!isBound())
+            return "ServerSocket[unbound]";
+        return "ServerSocket[addr=" + getInetAddress() +
+                ",localport=" + getLocalPort()  + "]";
+    }
+
+    public void setReceiveBufferSize(int size) throws SocketException {
+        if (size <= 0)
+            throw new IllegalArgumentException("size can not be 0 or negative");
+        try {
+            getChannelImpl().setOption(StandardSocketOptions.SO_RCVBUF, size);
+        } catch (IOException x) {
+            Net.translateToSocketException(x);
+        }
+    }
+
+    public int getReceiveBufferSize() throws SocketException {
+        try {
+            return getChannelImpl().getOption(StandardSocketOptions.SO_RCVBUF);
+        } catch (IOException x) {
+            Net.translateToSocketException(x);
+            return -1;          // Never happens
+        }
+    }
+
+    private ServerSocketChannelImpl getChannelImpl() throws SocketException {
+        if (ssc == null) {
+            try {
+                ssc = (ServerSocketChannelImpl) ServerSocketChannel.open();
+                ssc.configureBlocking(false);
+            } catch (IOException e) {
+                throw new SocketException(e.getMessage());
+            }
+        }
+        return ssc;
+    }
+}
diff --git a/src/java.base/share/classes/sun/nio/ch/WispSocketImpl.java b/src/java.base/share/classes/sun/nio/ch/WispSocketImpl.java
new file mode 100644
index 00000000000..03bf5db146a
--- /dev/null
+++ b/src/java.base/share/classes/sun/nio/ch/WispSocketImpl.java
@@ -0,0 +1,378 @@
+package sun.nio.ch;
+
+import com.alibaba.wisp.util.io.WispInputStream;
+import com.alibaba.wisp.util.io.WispOutputStream;
+import jdk.internal.access.SharedSecrets;
+import jdk.internal.access.WispEngineAccess;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.net.*;
+import java.nio.channels.SelectionKey;
+import java.nio.channels.SocketChannel;
+import java.security.AccessController;
+import java.security.PrivilegedExceptionAction;
+import java.util.concurrent.TimeUnit;
+
+
+// Make a socket channel be like a socket and yield on block
+
+public class WispSocketImpl
+{
+    private static WispEngineAccess WEA = SharedSecrets.getWispEngineAccess();
+    // The channel being adapted
+    private SocketChannelImpl sc = null;
+    // 1 verse 1 related socket
+    private Socket so;
+    // Timeout "option" value for reads
+    protected int timeout = 0;
+    private InputStream socketInputStream = null;
+
+    public WispSocketImpl(Socket so) {
+        this.so = so;
+    }
+
+    public WispSocketImpl(SocketChannel sc, Socket so) {
+        this.so = so;
+        this.sc = (SocketChannelImpl) sc;
+    }
+
+    public SocketChannel getChannel() {
+        return sc;
+    }
+
+    // Override this method just to protect against changes in the superclass
+    //
+    public void connect(SocketAddress remote) throws IOException {
+        connect(remote, 0);
+    }
+
+    public void connect(SocketAddress remote, int timeout) throws IOException {
+        if (remote == null)
+            throw new IllegalArgumentException("connect: The address can't be null");
+        if (timeout < 0)
+            throw new IllegalArgumentException("connect: timeout can't be negative");
+
+        final SocketChannel ch = getChannelImpl();
+        try {
+            if (ch.connect(remote)) return;
+
+            if (timeout > 0)
+                WEA.addTimer(System.nanoTime() + TimeUnit.MILLISECONDS.toNanos(timeout));
+
+            do {
+                WEA.registerEvent(ch, SelectionKey.OP_CONNECT);
+                WEA.park(-1);
+
+                if (timeout > 0 && WEA.isTimeout()) {
+                    throw new SocketTimeoutException("time out");
+                }
+            } while (!ch.finishConnect());
+
+        } catch (Exception x) {
+            // see AbstractPlainSocketImpl#doConnect
+            try {
+                Net.translateException(x, true);
+            } catch (IOException e) {
+                sc.close();
+                throw e;
+            }
+        } finally {
+            if (timeout > 0) {
+                WEA.cancelTimer();
+            }
+
+            if (ch.isBlocking())
+                ch.configureBlocking(false);
+
+            WEA.unregisterEvent();
+        }
+    }
+
+    public void bind(SocketAddress local) throws IOException {
+        try {
+            getChannelImpl().bind(local);
+        } catch (Exception x) {
+            Net.translateException(x);
+        }
+    }
+
+    public InetAddress getInetAddress() {
+        SocketAddress remote = sc == null ? null : sc.remoteAddress();
+        if (remote == null) {
+            return null;
+        } else {
+            return ((InetSocketAddress)remote).getAddress();
+        }
+    }
+
+    public InetAddress getLocalAddress() {
+        SocketChannelImpl ch = null;
+        try {
+            ch = getChannelImpl();
+        } catch (SocketException e) {
+            // return 0.0.0.0
+        }
+        if (ch != null && ch.isOpen()) {
+            SocketAddress local = ch.localAddress();
+            if (local != null) {
+                return Net.getRevealedLocalAddress(local).getAddress();
+            }
+        }
+        return new InetSocketAddress(0).getAddress();
+    }
+
+    public int getPort() {
+        SocketAddress remote = sc == null ? null : sc.remoteAddress();
+        if (remote == null) {
+            return 0;
+        } else {
+            return ((InetSocketAddress)remote).getPort();
+        }
+    }
+
+    public int getLocalPort() {
+        SocketChannelImpl ch = null;
+        try {
+            ch = getChannelImpl();
+        } catch (SocketException e) {
+            // return 0.0.0.0
+        }
+        SocketAddress local = ch == null ? null : ch.localAddress();
+        if (local == null) {
+            return -1;
+        } else {
+            return ((InetSocketAddress)local).getPort();
+        }
+    }
+
+    @SuppressWarnings("removal")
+    public InputStream getInputStream() throws IOException {
+        if (isClosed())
+            throw new SocketException("Socket is closed");
+        if (!isConnected())
+            throw new SocketException("Socket is not connected");
+        if (isInputShutdown())
+            throw new SocketException("Socket input is shutdown");
+        if (socketInputStream == null) {
+            try {
+                socketInputStream = AccessController.doPrivileged(
+                    new PrivilegedExceptionAction() {
+                        public InputStream run() throws IOException {
+                            return new WispInputStream(getChannelImpl(), so);
+                        }
+                    });
+            } catch (java.security.PrivilegedActionException e) {
+                throw (IOException)e.getException();
+            }
+        }
+        return socketInputStream;
+    }
+
+    @SuppressWarnings("removal")
+    public OutputStream getOutputStream() throws IOException {
+        if (isClosed())
+            throw new SocketException("Socket is closed");
+        if (!isConnected())
+            throw new SocketException("Socket is not connected");
+        if (isOutputShutdown())
+            throw new SocketException("Socket output is shutdown");
+        try {
+            return AccessController.doPrivileged(
+                new PrivilegedExceptionAction() {
+                    public OutputStream run() throws IOException {
+                        return new WispOutputStream(getChannelImpl(), so);
+                    }
+                });
+        } catch (java.security.PrivilegedActionException e) {
+            throw (IOException)e.getException();
+        }
+    }
+
+    private void setBooleanOption(SocketOption name, boolean value)
+        throws SocketException {
+        try {
+            getChannelImpl().setOption(name, value);
+        } catch (IOException x) {
+            Net.translateToSocketException(x);
+        }
+    }
+
+    private void setIntOption(SocketOption name, int value)
+        throws SocketException {
+        try {
+            getChannelImpl().setOption(name, value);
+        } catch (IOException x) {
+            Net.translateToSocketException(x);
+        }
+    }
+
+    private boolean getBooleanOption(SocketOption name) throws SocketException {
+        try {
+            return getChannelImpl().getOption(name);
+        } catch (IOException x) {
+            Net.translateToSocketException(x);
+            return false;       // keep compiler happy
+        }
+    }
+
+    private int getIntOption(SocketOption name) throws SocketException {
+        try {
+            return getChannelImpl().getOption(name);
+        } catch (IOException x) {
+            Net.translateToSocketException(x);
+            return -1;          // keep compiler happy
+        }
+    }
+
+    public void setTcpNoDelay(boolean on) throws SocketException {
+        setBooleanOption(StandardSocketOptions.TCP_NODELAY, on);
+    }
+
+    public boolean getTcpNoDelay() throws SocketException {
+        return getBooleanOption(StandardSocketOptions.TCP_NODELAY);
+    }
+
+    public void setSoLinger(boolean on, int linger) throws SocketException {
+        if (!on)
+            linger = -1;
+        setIntOption(StandardSocketOptions.SO_LINGER, linger);
+    }
+
+    public int getSoLinger() throws SocketException {
+        return getIntOption(StandardSocketOptions.SO_LINGER);
+    }
+
+    public void sendUrgentData(int data) throws IOException {
+        int n = getChannelImpl().sendOutOfBandData((byte) data);
+        if (n == 0)
+            throw new IOException("Socket buffer full");
+    }
+
+    public void setOOBInline(boolean on) throws SocketException {
+        setBooleanOption(ExtendedSocketOption.SO_OOBINLINE, on);
+    }
+
+    public boolean getOOBInline() throws SocketException {
+        return getBooleanOption(ExtendedSocketOption.SO_OOBINLINE);
+    }
+
+    public void setSoTimeout(int timeout) throws SocketException {
+        if (timeout < 0)
+            throw new IllegalArgumentException("timeout can't be negative");
+        this.timeout = timeout;
+    }
+
+    public int getSoTimeout() throws SocketException {
+        return timeout;
+    }
+
+    public void setSendBufferSize(int size) throws SocketException {
+        // size 0 valid for SocketChannel, invalid for Socket
+        if (size <= 0)
+            throw new IllegalArgumentException("Invalid send size");
+        setIntOption(StandardSocketOptions.SO_SNDBUF, size);
+    }
+
+    public int getSendBufferSize() throws SocketException {
+        return getIntOption(StandardSocketOptions.SO_SNDBUF);
+    }
+
+    public void setReceiveBufferSize(int size) throws SocketException {
+        // size 0 valid for SocketChannel, invalid for Socket
+        if (size <= 0)
+            throw new IllegalArgumentException("Invalid receive size");
+        setIntOption(StandardSocketOptions.SO_RCVBUF, size);
+    }
+
+    public int getReceiveBufferSize() throws SocketException {
+        return getIntOption(StandardSocketOptions.SO_RCVBUF);
+    }
+
+    public void setKeepAlive(boolean on) throws SocketException {
+        setBooleanOption(StandardSocketOptions.SO_KEEPALIVE, on);
+    }
+
+    public boolean getKeepAlive() throws SocketException {
+        return getBooleanOption(StandardSocketOptions.SO_KEEPALIVE);
+    }
+
+    public void setTrafficClass(int tc) throws SocketException {
+        setIntOption(StandardSocketOptions.IP_TOS, tc);
+    }
+
+    public int getTrafficClass() throws SocketException {
+        return getIntOption(StandardSocketOptions.IP_TOS);
+    }
+
+    public void setReuseAddress(boolean on) throws SocketException {
+        setBooleanOption(StandardSocketOptions.SO_REUSEADDR, on);
+    }
+
+    public boolean getReuseAddress() throws SocketException {
+        return getBooleanOption(StandardSocketOptions.SO_REUSEADDR);
+    }
+
+    public void close() throws IOException {
+        if (sc != null) {
+            sc.close();
+        }
+    }
+
+    public void shutdownInput() throws IOException {
+        try {
+            getChannelImpl().shutdownInput();
+        } catch (Exception x) {
+            Net.translateException(x);
+        }
+    }
+
+    public void shutdownOutput() throws IOException {
+        try {
+            getChannelImpl().shutdownOutput();
+        } catch (Exception x) {
+            Net.translateException(x);
+        }
+    }
+
+    public String toString() {
+        if (isConnected())
+            return "Socket[addr=" + getInetAddress() +
+                ",port=" + getPort() +
+                ",localport=" + getLocalPort() + "]";
+        return "Socket[unconnected]";
+    }
+
+    public boolean isConnected() {
+        return sc != null && sc.isConnected();
+    }
+
+    public boolean isBound() {
+        return sc != null && sc.localAddress() != null;
+    }
+
+    public boolean isClosed() {
+        return sc != null && !sc.isOpen();
+    }
+
+    public boolean isInputShutdown() {
+        return sc != null && !sc.isInputOpen();
+    }
+
+    public boolean isOutputShutdown() {
+        return sc != null && !sc.isOutputOpen();
+    }
+
+    private SocketChannelImpl getChannelImpl() throws SocketException {
+        if (sc == null) {
+            try {
+                sc = (SocketChannelImpl) SocketChannel.open();
+                sc.configureBlocking(false);
+            } catch (IOException e) {
+                throw new SocketException(e.getMessage());
+            }
+        }
+        return sc;
+    }
+}
diff --git a/src/java.base/share/classes/sun/nio/ch/WispUdpSocketImpl.java b/src/java.base/share/classes/sun/nio/ch/WispUdpSocketImpl.java
new file mode 100644
index 00000000000..44119e4b1ee
--- /dev/null
+++ b/src/java.base/share/classes/sun/nio/ch/WispUdpSocketImpl.java
@@ -0,0 +1,348 @@
+package sun.nio.ch;
+
+
+import jdk.internal.access.SharedSecrets;
+import jdk.internal.access.WispEngineAccess;
+
+import java.io.IOException;
+import java.net.*;
+import java.nio.ByteBuffer;
+import java.nio.channels.DatagramChannel;
+import java.nio.channels.SelectionKey;
+import java.util.concurrent.TimeUnit;
+
+
+// Make a udp socket channel look like a datagram socket.
+
+public class WispUdpSocketImpl {
+
+    private static WispEngineAccess WEA = SharedSecrets.getWispEngineAccess();
+    // 1 verse 1 related socket
+    private DatagramChannelImpl dc;
+
+    private DatagramSocket so;
+
+    private volatile int timeout = 0;
+
+    public WispUdpSocketImpl(DatagramSocket so) {
+        this.so = so;
+    }
+
+    public WispUdpSocketImpl(DatagramChannel dc, DatagramSocket so) {
+        this.so = so;
+        this.dc = (DatagramChannelImpl) dc;
+    }
+
+    public void bind(SocketAddress local) throws SocketException {
+        try {
+            if (local == null)
+                local = new InetSocketAddress(0);
+            getChannelImpl().bind(local);
+        } catch (Exception x) {
+            Net.translateToSocketException(x);
+        }
+    }
+
+    public void connect(InetAddress address, int port) {
+        try {
+            connect(new InetSocketAddress(address, port));
+        } catch (SocketException x) {
+        }
+    }
+
+    public void connect(SocketAddress remote) throws SocketException {
+        if (remote == null)
+            throw new IllegalArgumentException("Address can't be null");
+
+        InetSocketAddress isa = Net.asInetSocketAddress(remote);
+        int port = isa.getPort();
+        if (port < 0 || port > 0xFFFF)
+            throw new IllegalArgumentException("connect: " + port);
+        if (remote == null)
+            throw new IllegalArgumentException("connect: null address");
+        if (isClosed())
+            return;
+        try {
+            getChannelImpl().connect(remote);
+        } catch (Exception x) {
+            Net.translateToSocketException(x);
+        }
+    }
+
+    public void disconnect() {
+        try {
+            getChannelImpl().disconnect();
+        } catch (IOException x) {
+            throw new Error(x);
+        }
+    }
+
+    public boolean isBound() {
+        return dc != null && dc.localAddress() != null;
+    }
+
+    public boolean isConnected() {
+        return dc != null && dc.remoteAddress() != null;
+    }
+
+    public InetAddress getInetAddress() {
+        return (isConnected()
+                ? Net.asInetSocketAddress(dc.remoteAddress()).getAddress()
+                : null);
+    }
+
+    public int getPort() {
+        return (isConnected()
+                ? Net.asInetSocketAddress(dc.remoteAddress()).getPort()
+                : -1);
+    }
+
+    public void send(DatagramPacket p) throws IOException {
+        final DatagramChannelImpl ch = getChannelImpl();
+        try {
+            ByteBuffer bb = ByteBuffer.wrap(p.getData(),
+                    p.getOffset(),
+                    p.getLength());
+
+            boolean useWrite = ch.isConnected() && p.getAddress() == null;
+            if (useWrite) {
+                InetSocketAddress isa = ch.remoteAddress();
+                p.setPort(isa.getPort());
+                p.setAddress(isa.getAddress());
+            }
+
+            int writeLen = bb.remaining();
+            if ((useWrite ? ch.write(bb) : ch.send(bb, p.getSocketAddress())) == writeLen)
+                return;
+
+            /* Don't call so.getSoTimeout() here as it will try to acquire the lock of datagram socket
+               which might be already held by receiver, hence deadlock happens */
+            if (getSoTimeout() > 0)
+                WEA.addTimer(System.nanoTime() + TimeUnit.MILLISECONDS.toNanos(getSoTimeout()));
+
+            do {
+                WEA.registerEvent(ch, SelectionKey.OP_WRITE);
+                WEA.park(-1);
+
+                if (getSoTimeout() > 0 && WEA.isTimeout()) {
+                    throw new SocketTimeoutException("time out");
+                }
+                if (useWrite) {
+                    ch.write(bb);
+                } else {
+                    ch.send(bb, p.getSocketAddress());
+                }
+            } while (bb.remaining() > 0);
+
+        } catch (IOException x) {
+            Net.translateException(x);
+        } finally {
+            if (getSoTimeout() > 0) {
+                WEA.cancelTimer();
+            }
+            WEA.unregisterEvent();
+        }
+    }
+
+
+    private SocketAddress receive(ByteBuffer bb) throws IOException {
+        final DatagramChannelImpl ch = getChannelImpl();
+        SocketAddress sa;
+
+        try {
+            if ((sa = ch.receive(bb)) != null)
+                return sa;
+
+            if (getSoTimeout() > 0)
+                WEA.addTimer(System.nanoTime() + TimeUnit.MILLISECONDS.toNanos(getSoTimeout()));
+
+            do {
+                WEA.registerEvent(ch, SelectionKey.OP_READ);
+                WEA.park(-1);
+
+                if (getSoTimeout() > 0 && WEA.isTimeout()) {
+                    throw new SocketTimeoutException("time out");
+                }
+            } while ((sa = ch.receive(bb)) == null);
+        } finally {
+            if (getSoTimeout() > 0) {
+                WEA.cancelTimer();
+            }
+            WEA.unregisterEvent();
+        }
+
+        return sa;
+    }
+
+    public int receive(DatagramPacket p, int bufLen) throws IOException {
+        int dataLen = 0;
+        try {
+            ByteBuffer bb = ByteBuffer.wrap(p.getData(),
+                    p.getOffset(),
+                    bufLen);
+            SocketAddress sender = receive(bb);
+            p.setSocketAddress(sender);
+            dataLen = bb.position() - p.getOffset();
+        } catch (IOException x) {
+            Net.translateException(x);
+        }
+        return dataLen;
+    }
+
+    @SuppressWarnings("removal")
+    public InetAddress getLocalAddress() {
+        if (isClosed())
+            return null;
+        DatagramChannelImpl ch = null;
+        try {
+            ch = getChannelImpl();
+        } catch (SocketException e) {
+            // return 0.0.0.0
+        }
+        SocketAddress local = ch == null ? null : ch.localAddress();
+        if (local == null)
+            local = new InetSocketAddress(0);
+        InetAddress result = ((InetSocketAddress) local).getAddress();
+        SecurityManager sm = System.getSecurityManager();
+        if (sm != null) {
+            try {
+                sm.checkConnect(result.getHostAddress(), -1);
+            } catch (SecurityException x) {
+                return new InetSocketAddress(0).getAddress();
+            }
+        }
+        return result;
+    }
+
+    public int getLocalPort() {
+        if (isClosed())
+            return -1;
+        try {
+            SocketAddress local = getChannelImpl().getLocalAddress();
+            if (local != null) {
+                return ((InetSocketAddress) local).getPort();
+            }
+        } catch (Exception x) {
+        }
+        return 0;
+    }
+
+    public void setSoTimeout(int timeout) throws SocketException {
+        this.timeout = timeout;
+    }
+
+    public int getSoTimeout() throws SocketException {
+        return timeout;
+    }
+
+    private void setBooleanOption(SocketOption name, boolean value)
+            throws SocketException {
+        try {
+            getChannelImpl().setOption(name, value);
+        } catch (IOException x) {
+            Net.translateToSocketException(x);
+        }
+    }
+
+    private void setIntOption(SocketOption name, int value)
+            throws SocketException {
+        try {
+            getChannelImpl().setOption(name, value);
+        } catch (IOException x) {
+            Net.translateToSocketException(x);
+        }
+    }
+
+    private boolean getBooleanOption(SocketOption name) throws SocketException {
+        try {
+            return getChannelImpl().getOption(name);
+        } catch (IOException x) {
+            Net.translateToSocketException(x);
+            return false;       // keep compiler happy
+        }
+    }
+
+    private int getIntOption(SocketOption name) throws SocketException {
+        try {
+            return getChannelImpl().getOption(name);
+        } catch (IOException x) {
+            Net.translateToSocketException(x);
+            return -1;          // keep compiler happy
+        }
+    }
+
+    public void setSendBufferSize(int size) throws SocketException {
+        if (size <= 0)
+            throw new IllegalArgumentException("Invalid send size");
+        setIntOption(StandardSocketOptions.SO_SNDBUF, size);
+    }
+
+    public int getSendBufferSize() throws SocketException {
+        return getIntOption(StandardSocketOptions.SO_SNDBUF);
+    }
+
+    public void setReceiveBufferSize(int size) throws SocketException {
+        if (size <= 0)
+            throw new IllegalArgumentException("Invalid receive size");
+        setIntOption(StandardSocketOptions.SO_RCVBUF, size);
+    }
+
+    public int getReceiveBufferSize() throws SocketException {
+        return getIntOption(StandardSocketOptions.SO_RCVBUF);
+    }
+
+    public void setReuseAddress(boolean on) throws SocketException {
+        setBooleanOption(StandardSocketOptions.SO_REUSEADDR, on);
+    }
+
+    public boolean getReuseAddress() throws SocketException {
+        return getBooleanOption(StandardSocketOptions.SO_REUSEADDR);
+
+    }
+
+    public void setBroadcast(boolean on) throws SocketException {
+        setBooleanOption(StandardSocketOptions.SO_BROADCAST, on);
+    }
+
+    public boolean getBroadcast() throws SocketException {
+        return getBooleanOption(StandardSocketOptions.SO_BROADCAST);
+    }
+
+    public void setTrafficClass(int tc) throws SocketException {
+        setIntOption(StandardSocketOptions.IP_TOS, tc);
+    }
+
+    public int getTrafficClass() throws SocketException {
+        return getIntOption(StandardSocketOptions.IP_TOS);
+    }
+
+    public void close() {
+        if (dc != null) {
+            try {
+                dc.close();
+            } catch (IOException x) {
+                throw new Error(x);
+            }
+        }
+    }
+
+    public boolean isClosed() {
+        return dc != null && !dc.isOpen();
+    }
+
+    public DatagramChannel getChannel() {
+        return dc;
+    }
+
+    private DatagramChannelImpl getChannelImpl() throws SocketException {
+        if (dc == null) {
+            try {
+                dc = (DatagramChannelImpl) DatagramChannel.open();
+                dc.configureBlocking(false);
+            } catch (IOException e) {
+                throw new SocketException(e.getMessage());
+            }
+        }
+        return dc;
+    }
+}
diff --git a/src/java.base/share/conf/wisp.properties b/src/java.base/share/conf/wisp.properties
new file mode 100644
index 00000000000..4b4bf2b2e33
--- /dev/null
+++ b/src/java.base/share/conf/wisp.properties
@@ -0,0 +1,19 @@
+############################################################
+#               Wisp Configuration File
+############################################################
+#
+# This properties file is used to specify the default
+# behavior when com.alibaba.shiftThreadModel is on
+
+com.alibaba.wisp.biz.selector=com.taobao.gecko.core.util.SystemUtils::openSelector;\
+  com.google.code.yanf4j.util.SystemUtils::openSelector
+com.alibaba.wisp.biz.manage=org.apache.tomcat.util.net.JIoEndpoint::processSocket;\
+  io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe::read;\
+  com.taobao.gecko.core.nio.impl.SelectorManager::start;\
+  com.google.code.yanf4j.nio.impl.SelectorManager::start
+com.alibaba.wisp.biz.current=java.dyn.CoroutineBase::startInternal
+com.alibaba.wisp.biz.selector.sleep=io.netty.channel.nio.NioEventLoop::openSelector;\
+  org.apache.mina.transport.socket.nio.SocketIoProcessor::startupWorker
+com.alibaba.wisp.biz.black=io.netty.util.concurrent.SingleThreadEventExecutor::startThread;\
+  org.apache.mina.transport.socket.nio.SocketIoProcessor::startupWorker;\
+  org.apache.mina.core.polling.AbstractPollingIoConnector::startupWorker
diff --git a/src/java.base/share/native/libjava/Coroutine.c b/src/java.base/share/native/libjava/Coroutine.c
new file mode 100644
index 00000000000..c38cb783452
--- /dev/null
+++ b/src/java.base/share/native/libjava/Coroutine.c
@@ -0,0 +1,18 @@
+#include "jni.h"
+#include "jvm.h"
+#include "java_dyn_Coroutine.h"
+
+#define ARRAY_LENGTH(a) (sizeof(a)/sizeof(a[0]))
+
+#define LANG "Ljava/lang/"
+#define OBJ LANG"Object;"
+
+static JNINativeMethod methods[] = {
+  {"setWispTask","(JI"OBJ OBJ")V", (void *)&JVM_SetWispTask},
+};
+
+JNIEXPORT void JNICALL
+Java_java_dyn_Coroutine_registerNatives(JNIEnv *env, jclass cls)
+{
+    (*env)->RegisterNatives(env, cls, methods, ARRAY_LENGTH(methods));
+}
diff --git a/src/java.base/share/native/libjava/Thread.c b/src/java.base/share/native/libjava/Thread.c
index 3f176f814e4..2e5441f4171 100644
--- a/src/java.base/share/native/libjava/Thread.c
+++ b/src/java.base/share/native/libjava/Thread.c
@@ -43,18 +43,19 @@
 static JNINativeMethod methods[] = {
     {"start0",           "()V",        (void *)&JVM_StartThread},
     {"stop0",            "(" OBJ ")V", (void *)&JVM_StopThread},
-    {"isAlive",          "()Z",        (void *)&JVM_IsThreadAlive},
+    {"isAlive0",         "()Z",        (void *)&JVM_IsThreadAlive},
     {"suspend0",         "()V",        (void *)&JVM_SuspendThread},
     {"resume0",          "()V",        (void *)&JVM_ResumeThread},
     {"setPriority0",     "(I)V",       (void *)&JVM_SetThreadPriority},
-    {"yield",            "()V",        (void *)&JVM_Yield},
-    {"sleep",            "(J)V",       (void *)&JVM_Sleep},
-    {"currentThread",    "()" THD,     (void *)&JVM_CurrentThread},
+    {"yield0",           "()V",        (void *)&JVM_Yield},
+    {"sleep0",           "(J)V",       (void *)&JVM_Sleep},
+    {"currentThread0",   "()" THD,     (void *)&JVM_CurrentThread},
     {"interrupt0",       "()V",        (void *)&JVM_Interrupt},
     {"holdsLock",        "(" OBJ ")Z", (void *)&JVM_HoldsLock},
     {"getThreads",        "()[" THD,   (void *)&JVM_GetAllThreads},
     {"dumpThreads",      "([" THD ")[[" STE, (void *)&JVM_DumpThreads},
     {"setNativeName",    "(" STR ")V", (void *)&JVM_SetNativeThreadName},
+    {"isInNative",       "()Z",        (void *)&JVM_IsInNative},
 };
 
 #undef THD
diff --git a/src/java.base/share/native/libjava/WispEngine.c b/src/java.base/share/native/libjava/WispEngine.c
new file mode 100644
index 00000000000..4d8433656da
--- /dev/null
+++ b/src/java.base/share/native/libjava/WispEngine.c
@@ -0,0 +1,15 @@
+#include "jni.h"
+#include "jvm.h"
+#include "com_alibaba_wisp_engine_WispEngine.h"
+
+#define ARRAY_LENGTH(a) (sizeof(a)/sizeof(a[0]))
+
+static JNINativeMethod methods[] = {
+  {"getProxyUnpark",  "([I)I", (void *)&JVM_GetProxyUnpark},
+};
+
+JNIEXPORT void JNICALL
+Java_com_alibaba_wisp_engine_WispEngine_registerNatives(JNIEnv *env, jclass cls)
+{
+    (*env)->RegisterNatives(env, cls, methods, ARRAY_LENGTH(methods));
+}
diff --git a/src/java.base/share/native/libjava/WispSysmon.c b/src/java.base/share/native/libjava/WispSysmon.c
new file mode 100644
index 00000000000..861a081e1fd
--- /dev/null
+++ b/src/java.base/share/native/libjava/WispSysmon.c
@@ -0,0 +1,15 @@
+#include "jni.h"
+#include "jvm.h"
+#include "com_alibaba_wisp_engine_WispSysmon.h"
+
+#define ARRAY_LENGTH(a) (sizeof(a)/sizeof(a[0]))
+
+static JNINativeMethod methods[] = {
+  {"markPreempt","(Ljava/lang/Thread;)V", (void *)&JVM_MarkPreempt},
+};
+
+JNIEXPORT void JNICALL
+Java_com_alibaba_wisp_engine_WispSysmon_registerNatives(JNIEnv *env, jclass cls)
+{
+    (*env)->RegisterNatives(env, cls, methods, ARRAY_LENGTH(methods));
+}
diff --git a/src/java.base/share/native/libjava/WispTask.c b/src/java.base/share/native/libjava/WispTask.c
new file mode 100644
index 00000000000..fd32fa6179e
--- /dev/null
+++ b/src/java.base/share/native/libjava/WispTask.c
@@ -0,0 +1,17 @@
+#include "jni.h"
+#include "jvm.h"
+#include "com_alibaba_wisp_engine_WispTask.h"
+
+#define ARRAY_LENGTH(a) (sizeof(a)/sizeof(a[0]))
+
+#define THD "Ljava/lang/Thread;"
+
+static JNINativeMethod methods[] = {
+        {"checkAndClearNativeInterruptForWisp",    "(" THD ")Z",       (void *)&JVM_CheckAndClearNativeInterruptForWisp},
+};
+
+JNIEXPORT void JNICALL
+Java_com_alibaba_wisp_engine_WispTask_registerNatives(JNIEnv *env, jclass cls)
+{
+  (*env)->RegisterNatives(env, cls, methods, ARRAY_LENGTH(methods));
+}
diff --git a/test/hotspot/jtreg/runtime/coroutine/C1ThrowSyncExceptionTest.java b/test/hotspot/jtreg/runtime/coroutine/C1ThrowSyncExceptionTest.java
new file mode 100644
index 00000000000..686fc28ee28
--- /dev/null
+++ b/test/hotspot/jtreg/runtime/coroutine/C1ThrowSyncExceptionTest.java
@@ -0,0 +1,85 @@
+/*
+ * @test
+ * @summary test a special wisp unpark case for C1 compiled method
+ * @modules java.base/jdk.internal.misc
+ * @run main/othervm  -XX:-UseBiasedLocking -XX:+EnableCoroutine -XX:+UseWispMonitor -Dcom.alibaba.transparentAsync=true -XX:TieredStopAtLevel=1 C1ThrowSyncExceptionTest
+ */
+
+
+import com.alibaba.wisp.engine.WispEngine;
+import jdk.internal.access.SharedSecrets;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.locks.Condition;
+import java.util.concurrent.locks.Lock;
+import java.util.concurrent.locks.ReentrantLock;
+
+public class C1ThrowSyncExceptionTest {
+    final static Runner[] runners = new Runner[16];
+    static boolean JUC;
+    static final int N = 40000;
+
+    public static void main(String[] args) throws Exception {
+        doTest();
+    }
+
+    private static void doTest() {
+        for (int i = 0; i < runners.length; i++) {
+            runners[i] = new Runner(i);
+        }
+        List plans = new ArrayList<>(Arrays.asList(
+                () -> { // only coroutine
+                    for (Runner runner1 : runners) {
+                        WispEngine.dispatch(runner1);
+                    }
+                }));
+        Collections.shuffle(plans);
+
+        plans.forEach(plan -> {
+            finishLatch = new CountDownLatch(runners.length);
+            current = 0;
+            long start = System.currentTimeMillis();
+            plan.run(); // create runners
+            System.out.println("cost = " + (System.currentTimeMillis() - start) + "ms");
+        });
+
+        System.out.println("-----------");
+    }
+
+
+    static volatile int current = 0;
+    static final Lock lock = new ReentrantLock();
+    static final Condition cond = lock.newCondition();
+    static CountDownLatch finishLatch;
+
+    static class Runner implements Runnable {
+
+        private final int ord;
+
+        public Runner(int ord) {
+            this.ord = ord;
+        }
+        public synchronized void testThrowException() {
+            int[] memory = new int[10000];
+            int index = 10003;
+            // trigger null check exception
+            memory[index] = 5;
+        }
+        @Override
+        public void run() {
+            while (current < N) {
+                try {
+                    testThrowException();
+                } catch (RuntimeException e) {
+                    if (++current % 1000 == 0)
+                        System.out.println(SharedSecrets.getJavaLangAccess().currentThread0().getName() + "\t" + current);                            
+                }
+            }
+        }
+    }
+}
+
diff --git a/test/hotspot/jtreg/runtime/coroutine/CoroutineNativeConcurrentTest.java b/test/hotspot/jtreg/runtime/coroutine/CoroutineNativeConcurrentTest.java
new file mode 100644
index 00000000000..529d7b0c23b
--- /dev/null
+++ b/test/hotspot/jtreg/runtime/coroutine/CoroutineNativeConcurrentTest.java
@@ -0,0 +1,136 @@
+/* @test
+ * @summary unit tests for coroutine steal in concurrent situation
+ * @run junit/othervm/timeout=300 -XX:+EnableCoroutine CoroutineNativeConcurrentTest
+ */
+
+import org.junit.Test;
+
+import java.dyn.Coroutine;
+import java.dyn.CoroutineBase;
+import java.lang.reflect.InvocationTargetException;
+import java.security.AccessController;
+import java.security.PrivilegedAction;
+import java.util.concurrent.ThreadLocalRandom;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.lang.reflect.Method;
+import java.lang.reflect.Field;
+
+public class CoroutineNativeConcurrentTest {
+
+    class Invoker {
+        void testMethod() {
+            try {
+                Coroutine.yield();
+            } catch (Throwable e) {
+                e.printStackTrace();
+            }
+        }
+    }
+
+    class Constructor {
+        Constructor() {
+            Coroutine.yield();
+        }
+    }
+
+    public void doPrivilegeTest() {
+        PrivilegedAction paaa = () -> {
+            PrivilegedAction paa = () -> {
+                PrivilegedAction pa = () -> {     // lambda$0
+                    Coroutine.yield();
+                    return 0;
+                };
+                return 0f;
+            };
+            AccessController.doPrivileged(paa);
+            return 0.0d;
+        };
+        AccessController.doPrivileged(paaa);
+    }
+
+    public void doInvokeTest() {
+        Invoker in = new Invoker();
+        try {
+            Method method = Invoker.class.getDeclaredMethod("testMethod");
+            method.invoke(in);
+        } catch (NoSuchMethodException e) {
+            e.printStackTrace();
+        } catch (IllegalAccessException e) {
+            e.printStackTrace();
+        } catch (InvocationTargetException e) {
+            e.printStackTrace();
+        }
+    }
+
+    public void doConstructTest() {
+        try {
+            Class c = Class.forName("CoroutineNativeConcurrentTest$Constructor");
+            c.newInstance();
+        } catch (ClassNotFoundException e) {
+            e.printStackTrace();
+        } catch (IllegalAccessException e) {
+            e.printStackTrace();
+        } catch (InstantiationException e) {
+            e.printStackTrace();
+        }
+    }
+
+    @Test
+    public void randomSteal() {
+        final int THREADS = 10;
+        final int CORO_PER_TH = 10;
+        Coroutine[] coro = new Coroutine[THREADS * CORO_PER_TH];
+
+        for (int i = 0; i < CORO_PER_TH; i++) {
+            int idx = i;
+            coro[i] = new Coroutine(() -> {
+                while (true) {
+                    Coroutine.yield();
+                }
+            });
+        }
+
+        AtomicInteger sync = new AtomicInteger();
+
+        for (int th = 1; th < THREADS; th++) {
+            int cth = th;
+            Thread t = new Thread(() -> {
+                for (int i = 0; i < CORO_PER_TH; i++) {
+                    int idx = i;
+                    coro[CORO_PER_TH * cth + i] = new Coroutine(() -> {
+                        while (true) {
+                            doPrivilegeTest();
+                            doInvokeTest();
+                            doConstructTest();
+                        }
+                    });
+                }
+                sync.incrementAndGet();
+                while (sync.get() != THREADS) {}
+
+                runRandomCoroutines(System.nanoTime(), coro);
+            }, "randomSteal-" + th);
+            t.setDaemon(true);
+            t.start();
+        }
+
+        sync.incrementAndGet();
+        while (sync.get() != THREADS) {}
+    }
+
+    private static void runRandomCoroutines(long start, Coroutine[] coro) {
+        while (System.nanoTime() - start < TimeUnit.SECONDS.toNanos(5)) {
+            int n = ThreadLocalRandom.current().nextInt(coro.length);
+            Coroutine co = coro[n];
+            if (co.getThread() != Thread.currentThread()) {
+                co.steal(false);
+            }
+            Coroutine.yieldTo(co);
+        }
+    }
+
+    public static void main(String[] args) {
+        new CoroutineNativeConcurrentTest().randomSteal();
+    }
+}
diff --git a/test/hotspot/jtreg/runtime/coroutine/DirectUnparkTest.java b/test/hotspot/jtreg/runtime/coroutine/DirectUnparkTest.java
new file mode 100644
index 00000000000..8fa388635db
--- /dev/null
+++ b/test/hotspot/jtreg/runtime/coroutine/DirectUnparkTest.java
@@ -0,0 +1,55 @@
+/*
+ * @test
+ * @library /test/lib
+ * @summary Test the optimization of direct unpark with Object.wait/notify
+ * @run main/othervm  -XX:-UseBiasedLocking -XX:+EnableCoroutine -XX:+UseWispMonitor -Dcom.alibaba.wisp.transparentWispSwitch=true -Dcom.alibaba.wisp.enableThreadAsWisp=true -Dcom.alibaba.wisp.version=2 -Dcom.alibaba.wisp.allThreadAsWisp=true DirectUnparkTest
+ * @run main/othervm  -XX:-UseBiasedLocking -XX:+EnableCoroutine -XX:+UseWispMonitor -Dcom.alibaba.wisp.transparentWispSwitch=true -Dcom.alibaba.wisp.enableThreadAsWisp=true -Dcom.alibaba.wisp.version=2 DirectUnparkTest
+*/
+
+import com.alibaba.wisp.engine.WispEngine;
+
+import java.util.concurrent.CountDownLatch;
+
+import static jdk.test.lib.Asserts.*;
+
+public class DirectUnparkTest {
+    public static void main(String[] args) throws Exception {
+        DirectUnparkTest s = new DirectUnparkTest();
+        WispEngine.dispatch(s::bar);
+        long now = System.currentTimeMillis();
+        synchronized (s) {
+            s.latch.countDown();
+            while (s.finishCnt < N) {
+                s.wait();
+                s.finishCnt++;
+                s.notifyAll();
+            }
+        }
+        long elapsed = System.currentTimeMillis() - now;
+        if (elapsed > 3000)
+            throw new Error("elapsed = " + elapsed);
+    }
+
+    static volatile int finishCnt = 0;
+    private CountDownLatch latch = new CountDownLatch(1);
+    static final int N = 40000;
+
+    private void bar() {
+        try {
+            latch.await();
+        } catch (InterruptedException e) {
+            e.printStackTrace();
+        }
+        while (finishCnt < N) {
+            synchronized (this) {
+                notifyAll();
+                finishCnt++;
+                try {
+                    wait();
+                } catch (InterruptedException e) {
+                    e.printStackTrace();
+                }
+            }
+        }
+    }
+}
diff --git a/test/hotspot/jtreg/runtime/coroutine/InterruptedWaitTest.java b/test/hotspot/jtreg/runtime/coroutine/InterruptedWaitTest.java
new file mode 100644
index 00000000000..02943ed8196
--- /dev/null
+++ b/test/hotspot/jtreg/runtime/coroutine/InterruptedWaitTest.java
@@ -0,0 +1,34 @@
+/*
+ * @test
+ * @summary test obj.wait() could be interrupted
+ * @library /test/lib
+ * @run main/othervm -XX:+EnableCoroutine -XX:+UseWispMonitor -Dcom.alibaba.wisp.transparentWispSwitch=true InterruptedWaitTest
+ * @run main/othervm -XX:+EnableCoroutine -XX:+UseWispMonitor -Dcom.alibaba.wisp.transparentWispSwitch=true -Dcom.alibaba.wisp.version=2 InterruptedWaitTest
+ */
+
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import static jdk.test.lib.Asserts.*;
+
+public class InterruptedWaitTest {
+    public static void main(String[] args) throws Exception {
+        AtomicBoolean interrupted = new AtomicBoolean(false);
+        Thread t = new Thread(() -> {
+            synchronized (args) {
+                try {
+                    args.wait();
+                } catch (InterruptedException e) {
+                    interrupted.set(true);
+                }
+            }
+        });
+        t.start();
+        Thread.sleep(200L);
+        t.interrupt();
+        Thread.sleep(200L);
+
+
+        assertTrue(interrupted.get());
+    }
+}
+
diff --git a/test/hotspot/jtreg/runtime/coroutine/JStackTest.java b/test/hotspot/jtreg/runtime/coroutine/JStackTest.java
new file mode 100644
index 00000000000..6b108fb4ec6
--- /dev/null
+++ b/test/hotspot/jtreg/runtime/coroutine/JStackTest.java
@@ -0,0 +1,133 @@
+/*
+ * @test
+ * @library /test/lib
+ * @summary Test jstack steal counter
+ * @modules java.base/com.alibaba.wisp.engine:+open
+ * @modules java.base/java.dyn:+open
+ * @run main/othervm -XX:+EnableCoroutine -XX:+UseWispMonitor -Dcom.alibaba.transparentAsync=true -Dcom.alibaba.wisp.version=2 -XX:ActiveProcessorCount=2 JStackTest
+ */
+
+import com.alibaba.wisp.engine.WispEngine;
+import com.alibaba.wisp.engine.WispTask;
+
+import java.dyn.CoroutineBase;
+import java.io.BufferedReader;
+import java.io.InputStreamReader;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.locks.ReentrantLock;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;
+import java.lang.reflect.Field;
+
+import static jdk.test.lib.Asserts.*;
+
+public class JStackTest {
+
+    private static Field wisptask = null;
+    private static Field stealCount = null;
+    private static Field ctx = null;
+    private static Field data = null;
+
+    static {
+        try {
+            wisptask = WispEngine.class.getDeclaredField("current");
+            wisptask.setAccessible(true);
+        } catch (NoSuchFieldException e) {
+            e.printStackTrace();
+        }
+        try {
+            stealCount = WispTask.class.getDeclaredField("stealCount");
+            stealCount.setAccessible(true);
+        } catch (NoSuchFieldException e) {
+            e.printStackTrace();
+        }
+        try {
+            ctx = WispTask.class.getDeclaredField("ctx");
+            ctx.setAccessible(true);
+        } catch (NoSuchFieldException e) {
+            e.printStackTrace();
+        }
+        try {
+            data = CoroutineBase.class.getDeclaredField("data");
+            data.setAccessible(true);
+        } catch (NoSuchFieldException e) {
+            e.printStackTrace();
+        }
+    }
+
+    private static Map map = new HashMap<>();
+
+    private static void test() throws Exception {
+        ReentrantLock lock = new ReentrantLock();
+        lock.lock();
+        for (int i = 0; i < 100; i++) {
+            WispEngine.dispatch(() -> {
+                Object o = new Object();
+                synchronized (o) {
+                    lock.lock(); // block until outter call unlock()
+                    try {
+                        WispTask current = (WispTask) wisptask.get(WispEngine.current());
+                        long coroutine = (Long) data.get(ctx.get(current));
+                        int count = (Integer) stealCount.get(current);
+                        map.put("0x"+Long.toHexString(coroutine), count);
+                    } catch (IllegalAccessException e) {
+                        e.printStackTrace();
+                    }
+                    lock.unlock();
+                }
+            });
+        }
+
+        Thread.sleep(100);
+        lock.unlock();
+
+        List result = jstack();
+
+        int i = 0;
+        for (; i < result.size(); i++) {
+            if (result.get(i).contains("- Coroutine [")) {
+                String str = result.get(i);
+                String coroutine = matchCoroutine(str);
+                int stealCount = Integer.parseInt(matchStealCount(str));
+                assertTrue(map.get(coroutine) == stealCount);
+            }
+        }
+    }
+
+    private static String matchCoroutine(String str) {
+        Pattern pattern = Pattern.compile(".*\\[(.*)\\].*");
+        Matcher matcher = pattern.matcher(str);
+        if (matcher.find()) {
+            return matcher.group(1);
+        }
+        throw new RuntimeException("ShouldNotReachHere");
+    }
+
+    private static String matchStealCount(String str) {
+        Pattern pattern = Pattern.compile(".*steal=(\\d+).*");
+        Matcher matcher = pattern.matcher(str);
+        if (matcher.find()) {
+            return matcher.group(1);
+        }
+        throw new RuntimeException("ShouldNotReachHere");
+    }
+
+    private static List jstack() throws Exception {
+        List statusLines = Files.readAllLines(Paths.get("/proc/self/status"));
+        String pidLine = statusLines.stream().filter(l -> l.startsWith("Pid:")).findAny().orElse("1 -1");
+        int pid = Integer.valueOf(pidLine.split("\\s+")[1]);
+
+        Process p = Runtime.getRuntime().exec(System.getProperty("java.home") + "/bin/jstack " + pid);
+        List result = new BufferedReader(new InputStreamReader(p.getInputStream())).lines().collect(Collectors.toList());
+        return result;
+    }
+
+    public static void main(String[] args) throws Exception {
+        test();
+    }
+}
diff --git a/test/hotspot/jtreg/runtime/coroutine/MultiCoroutineStackTest.java b/test/hotspot/jtreg/runtime/coroutine/MultiCoroutineStackTest.java
new file mode 100644
index 00000000000..5e013cedc6e
--- /dev/null
+++ b/test/hotspot/jtreg/runtime/coroutine/MultiCoroutineStackTest.java
@@ -0,0 +1,133 @@
+/*
+ * @test
+ * @library /test/lib
+ * @summary Test jstack coroutine output
+ * @run main/othervm -XX:+EnableCoroutine -Dcom.alibaba.transparentAsync=true -XX:+UseWispMonitor MultiCoroutineStackTest
+ */
+
+import com.alibaba.wisp.engine.WispEngine;
+
+import java.io.BufferedReader;
+import java.io.InputStreamReader;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.util.List;
+import java.util.concurrent.locks.LockSupport;
+import java.util.concurrent.locks.ReentrantLock;
+import java.util.stream.Collectors;
+
+import static jdk.test.lib.Asserts.*;
+
+public class MultiCoroutineStackTest {
+    public static void main(String[] args) throws Exception {
+        baseTest();
+        testCoroutineName();
+        testParkingObj();
+        testWaitingToLock();
+    }
+
+    private static void testCoroutineName() throws Exception {
+        final String NAME = "test-coroutine-name";
+        ReentrantLock lock = new ReentrantLock();
+        lock.lock();
+        WispEngine.dispatch(() -> {
+            Thread.currentThread().setName(NAME);
+            lock.lock(); // block until outter call unlock()
+            lock.unlock();
+        });
+        boolean success = false;
+        for (String line : jstack()) {
+            if (line.contains("- Coroutine [") && line.contains(NAME)) {
+                success = true;
+                break;
+            }
+        }
+        assertTrue(success, "coroutine name not found");
+        lock.unlock();
+    }
+
+    private static void testParkingObj() throws Exception {
+        Thread[] coro = new Thread[1];
+
+        WispEngine.dispatch(() -> {
+            coro[0] = Thread.currentThread();
+            LockSupport.park(new MultiCoroutineStackTest());
+        });
+
+        boolean success = false;
+        for (String line : jstack()) {
+            if (line.contains("parking to wait for") && line.contains(MultiCoroutineStackTest.class.getSimpleName())) {
+                success = true;
+                break;
+            }
+        }
+        assertTrue(success, "\"parking to wait for\" not found");
+        LockSupport.unpark(coro[0]);
+    }
+
+    private static void testWaitingToLock() throws Exception {
+        Object lock = new MultiCoroutineStackTest();
+
+        synchronized (lock) {
+            WispEngine.dispatch(() -> {
+                synchronized (lock) {
+                }
+            });
+            boolean success = false;
+            for (String line : jstack()) {
+                if (line.contains("waiting to lock") && line.contains(MultiCoroutineStackTest.class.getSimpleName())) {
+                    success = true;
+                    break;
+                }
+            }
+            assertTrue(success, "\"waiting to lock\" not found");
+        }
+    }
+
+    private static void baseTest() throws Exception {
+        ReentrantLock lock = new ReentrantLock();
+        lock.lock();
+        WispEngine.dispatch(() -> {
+            Object o = new Object();
+            synchronized (o) {
+                lock.lock(); // block until outter call unlock()
+                lock.unlock();
+            }
+        });
+
+        List result = jstack();
+
+        int i = 0;
+        for (; i < result.size(); i++) {
+            if (result.get(i).contains("- Coroutine ["))
+                break;
+        }
+        assertTrue(i != result.size(), "coroutine stack not found");
+        assertTrue(result.get(i + 1).contains("java.dyn.CoroutineSupport.symmetricYieldTo"),
+                "unexpected stack top :" + result.get(i + 1));
+
+        boolean lockFound = false, jucFound = false;
+        for (String line : result) {
+            if (line.contains("java.util.concurrent.locks")) {
+                jucFound = true;
+            } else if (line.contains("- locked <0x")) {
+                lockFound = true;
+            }
+        }
+        assertTrue(lockFound, "synchronized not found");
+        assertTrue(jucFound, "j.u.c not found");
+
+        lock.unlock();
+    }
+
+    private static List jstack() throws Exception {
+        List statusLines = Files.readAllLines(Paths.get("/proc/self/status"));
+        String pidLine = statusLines.stream().filter(l -> l.startsWith("Pid:")).findAny().orElse("1 -1");
+        int pid = Integer.valueOf(pidLine.split("\\s+")[1]);
+
+        Process p = Runtime.getRuntime().exec(System.getProperty("java.home") + "/bin/jstack " + pid);
+        List result = new BufferedReader(new InputStreamReader(p.getInputStream())).lines().collect(Collectors.toList());
+        System.out.println(result.stream().collect(Collectors.joining("\n")));
+        return result;
+    }
+}
diff --git a/test/hotspot/jtreg/runtime/coroutine/PremainWithWispMonitorTest.java b/test/hotspot/jtreg/runtime/coroutine/PremainWithWispMonitorTest.java
new file mode 100644
index 00000000000..b82ba5e04c1
--- /dev/null
+++ b/test/hotspot/jtreg/runtime/coroutine/PremainWithWispMonitorTest.java
@@ -0,0 +1,41 @@
+/*
+ * @test
+ * @summary Test the fix that unpark might not be handled in WispThread::unpark due to due to WispEngine of main thread not properly been initialized in premain().
+ *
+ * @run shell premainWithWispMonitorTest.sh
+ */
+
+import com.alibaba.wisp.engine.WispEngine;
+
+import java.lang.instrument.Instrumentation;
+
+public class PremainWithWispMonitorTest {
+
+    private static Object lock = new Object();
+
+    public static void main(String[] args) throws Exception {
+        // BUG: before fix, WispEngine is not registered to jvm data structure
+        Thread t = new Thread(() -> {
+            synchronized (lock) { // blocked by current main thread
+            }
+        });
+        synchronized (lock) {
+            t.start();
+            Thread.sleep(100);
+        }
+        // unlock, if without fix, the handling for t's unpark will be missed in WispThread::unpark.
+        // The reason here is WispEngine instance is not correctly initialized for current main thread 
+        // in premain phase, which is triggered by JvmtiExport::post_vm_initialized
+
+        t.join(100);
+        if (t.isAlive()) {
+            throw new Error("lost unpark");
+        }
+    }
+
+    public static void premain (String agentArgs, Instrumentation instArg) {
+        // init WispEngine before initializeCoroutineSupport
+        WispEngine.current();
+    }
+}
+
diff --git a/test/hotspot/jtreg/runtime/coroutine/SimpleWispTest.java b/test/hotspot/jtreg/runtime/coroutine/SimpleWispTest.java
new file mode 100644
index 00000000000..b3fe1959c39
--- /dev/null
+++ b/test/hotspot/jtreg/runtime/coroutine/SimpleWispTest.java
@@ -0,0 +1,13 @@
+import com.alibaba.wisp.engine.WispEngine;
+
+public class SimpleWispTest {
+
+    public static void main(String[] args) {
+
+        WispEngine.dispatch(() -> {
+
+        });
+
+    }
+
+}
diff --git a/test/hotspot/jtreg/runtime/coroutine/TestAvoidDeoptCoroutineMethod.java b/test/hotspot/jtreg/runtime/coroutine/TestAvoidDeoptCoroutineMethod.java
new file mode 100644
index 00000000000..5d01c2ff442
--- /dev/null
+++ b/test/hotspot/jtreg/runtime/coroutine/TestAvoidDeoptCoroutineMethod.java
@@ -0,0 +1,33 @@
+/*
+ * @test TestAvoidDeoptCoroutineMethod
+ * @library /test/lib
+ * @build sun.hotspot.WhiteBox
+ * @run driver ClassFileInstaller sun.hotspot.WhiteBox
+ * @build TestAvoidDeoptCoroutineMethod
+ * @run main/othervm -XX:+EnableCoroutine -Xmx10m -Xms10m -Xbootclasspath/a:. -XX:+UnlockDiagnosticVMOptions -XX:+WhiteBoxAPI TestAvoidDeoptCoroutineMethod
+ * @summary test avoid coroutine intrinsic method to be deoptimized
+ */
+
+import sun.hotspot.WhiteBox;
+import java.dyn.Coroutine;
+import java.io.*;
+
+public class TestAvoidDeoptCoroutineMethod {
+    public static void main(String[] args) throws Exception {
+        WhiteBox whiteBox = WhiteBox.getWhiteBox();
+        runSomeCoroutines();
+        // deoptimize all
+        whiteBox.deoptimizeAll();
+        // if intrinsic methods of coroutine have been deoptimized, it will crash here
+        runSomeCoroutines();
+    }
+
+    private static void runSomeCoroutines() throws Exception {
+        for (int i = 0; i < 10000; i++) {
+            new Coroutine(() -> {});
+            Coroutine.yield(); // switch to new created coroutine and let it die
+        }
+        System.out.println("end of run");
+    }
+}
+
diff --git a/test/hotspot/jtreg/runtime/coroutine/TimeSliceSyncTest.java b/test/hotspot/jtreg/runtime/coroutine/TimeSliceSyncTest.java
new file mode 100644
index 00000000000..cf87fcc9257
--- /dev/null
+++ b/test/hotspot/jtreg/runtime/coroutine/TimeSliceSyncTest.java
@@ -0,0 +1,121 @@
+/*
+ * @test
+ * @library /testlibrary
+ * @summary This test ensures that coroutine time slice feature won't cause hang.
+ * @modules java.base/jdk.internal.misc
+ * @run main/othervm  -XX:-UseBiasedLocking -XX:+EnableCoroutine -XX:+UseWispMonitor -Dcom.alibaba.wisp.transparentWispSwitch=true -XX:-Inline -XX:+EnableCoroutineTimeSlice TimeSliceSyncTest
+ * @run main/othervm  -XX:-UseBiasedLocking -XX:+EnableCoroutine -XX:+UseWispMonitor -Dcom.alibaba.wisp.transparentWispSwitch=true -XX:-Inline -XX:+EnableCoroutineTimeSlice -Dcom.alibaba.wisp.version=2 TimeSliceSyncTest
+ */
+
+
+import com.alibaba.wisp.engine.WispEngine;
+import jdk.internal.access.SharedSecrets;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.locks.Condition;
+import java.util.concurrent.locks.Lock;
+import java.util.concurrent.locks.ReentrantLock;
+
+public class TimeSliceSyncTest {
+    final static Runner[] runners = new Runner[16];
+    static boolean JUC;
+    static final int N = 400000;
+
+    public static void main(String[] args) throws Exception {
+        doTest();
+    }
+
+    private static void doTest() {
+        for (int i = 0; i < runners.length; i++) {
+            runners[i] = new Runner(i);
+        }
+        List plans = new ArrayList<>(Arrays.asList(
+                () -> { // only coroutine
+                    for (Runner runner1 : runners) {
+                        WispEngine.dispatch(runner1);
+                    }
+                },
+                () -> { // only thread
+                    int n = 0;
+                    for (Runner runner : runners) {
+                        new Thread(runner, "MP-THREAD-RUNNER-" + n++).start();
+                    }
+                },
+                () -> { //mixed
+                    int n = 0;
+                    for (int i = 0; i < runners.length; i += 2) {
+                        final int ci = i;
+                        new Thread(() -> {
+                            for (int j = ci; j < ci + 2 && j < runners.length; j++) {
+                                WispEngine.dispatch(runners[j]);
+                            }
+                        }, "MP-MIX-RUNNER-" + n++).start();
+                    }
+                }));
+        Collections.shuffle(plans);
+
+        plans.forEach(plan -> {
+            finishLatch = new CountDownLatch(runners.length);
+            current = 0;
+            long start = System.currentTimeMillis();
+            plan.run(); // create runners
+            System.out.println("cost = " + (System.currentTimeMillis() - start) + "ms");
+        });
+
+        System.out.println("-----------");
+    }
+
+
+    static volatile int current = 0;
+    static final Lock lock = new ReentrantLock();
+    static final Condition cond = lock.newCondition();
+    static CountDownLatch finishLatch;
+
+    static class Runner implements Runnable {
+
+        private final int ord;
+
+        public Runner(int ord) {
+            this.ord = ord;
+        }
+        // The printX methods are synchronized, there is a monitorexit at the end of method
+        // If we perform yield before monitorexit, hang may happen.
+        public synchronized void print0() {
+            if ((current % 10000) == 0) {
+                System.out.println("print0 " + current);
+            }
+        }
+        public synchronized void print1() {
+            if ((current % 10000) == 0) {
+                System.out.println("print1 " + current);
+            }
+        }
+        public synchronized void print2() {
+            if ((current % 10000) == 0) {
+                System.out.println("print2 " + current);
+            }
+        } 
+        @Override
+        public void run() {
+            // 8 threads are created and execute synchronized methods.
+            // We must ensure that yield do not be excetued before monitorexit 
+            for (int i = 0; i < N; i++) {
+                int mod = current % 5;
+                if (mod == 0) {
+                    print0();
+                } else if (mod == 1) {
+                    print1();
+                } else {
+                    print2();
+                }
+                current++;
+            }
+        }
+    }
+}
+
diff --git a/test/hotspot/jtreg/runtime/coroutine/Wisp2SwitchTest.java b/test/hotspot/jtreg/runtime/coroutine/Wisp2SwitchTest.java
new file mode 100644
index 00000000000..6f7a9c0c951
--- /dev/null
+++ b/test/hotspot/jtreg/runtime/coroutine/Wisp2SwitchTest.java
@@ -0,0 +1,58 @@
+/*
+ * @test
+ * @summary test wisp2 switch
+ * @library /test/lib
+ * @modules java.base/jdk.internal.misc
+ * @modules java.base/com.alibaba.wisp.engine:+open
+ * @run main/othervm -XX:+UseWisp2  Wisp2SwitchTest
+ */
+
+
+import com.alibaba.wisp.engine.WispEngine;
+import jdk.internal.access.SharedSecrets;
+
+import java.lang.reflect.Field;
+import static jdk.test.lib.Asserts.*;
+
+public class Wisp2SwitchTest {
+    public static void main(String[] args) throws Exception {
+        WispEngine.dispatch(() -> {
+            for (int i = 0; i < 9999999; i++) {
+                try {
+                    Thread.sleep(100);
+                    System.out.println(i + ": " + SharedSecrets.getJavaLangAccess().currentThread0());
+                } catch (InterruptedException e) {
+                    e.printStackTrace();
+                }
+            }
+        });
+        System.out.println("Wisp2SwitchTest.main");
+        Field f = Class.forName("com.alibaba.wisp.engine.WispConfiguration").getDeclaredField("WISP_VERSION");
+        f.setAccessible(true);
+        int version = f.getInt(null);
+        assertTrue(version == 2, "Wisp Version isn't Wisp2");
+
+        boolean isEnabled;
+        f = Class.forName("com.alibaba.wisp.engine.WispConfiguration").getDeclaredField("TRANSPARENT_WISP_SWITCH");
+        f.setAccessible(true);
+        isEnabled = f.getBoolean(null);
+        assertTrue(isEnabled == true, "The property com.alibaba.wisp.transparentWispSwitch isn't enabled");
+
+        f = Class.forName("com.alibaba.wisp.engine.WispConfiguration").getDeclaredField("ENABLE_THREAD_AS_WISP");
+        f.setAccessible(true);
+        isEnabled = f.getBoolean(null);
+        assertTrue(isEnabled == true, "The property com.alibaba.wisp.enableThreadAsWisp isn't enabled");
+
+        f = Class.forName("com.alibaba.wisp.engine.WispConfiguration").getDeclaredField("ALL_THREAD_AS_WISP");
+        f.setAccessible(true);
+        isEnabled = f.getBoolean(null);
+        assertTrue(isEnabled == true, "The property com.alibaba.wisp.allThreadAsWisp isn't enabled");
+
+        f = Class.forName("com.alibaba.wisp.engine.WispConfiguration").getDeclaredField("ENABLE_HANDOFF");
+        f.setAccessible(true);
+        isEnabled = f.getBoolean(null);
+        assertFalse(isEnabled == true, "The property com.alibaba.wisp.enableHandOff isn't enabled");
+
+        Thread.sleep(1000);
+    }
+}
diff --git a/test/hotspot/jtreg/runtime/coroutine/Wisp2SwitchTest2.java b/test/hotspot/jtreg/runtime/coroutine/Wisp2SwitchTest2.java
new file mode 100644
index 00000000000..7baab36c447
--- /dev/null
+++ b/test/hotspot/jtreg/runtime/coroutine/Wisp2SwitchTest2.java
@@ -0,0 +1,59 @@
+/*
+ * @test
+ * @summary test XX:+UseWisp2 switch with -Dcom.alibaba.wisp.allThreadAsWisp=false
+ * @library /test/lib
+ * @modules java.base/jdk.internal.misc
+ * @modules java.base/com.alibaba.wisp.engine:+open
+ * @run main/othervm -XX:+UseWisp2 -Dcom.alibaba.wisp.allThreadAsWisp=false Wisp2SwitchTest2
+ */
+
+
+import com.alibaba.wisp.engine.WispEngine;
+import jdk.internal.access.SharedSecrets;
+
+import java.lang.reflect.Field;
+
+import static jdk.test.lib.Asserts.*;
+
+public class Wisp2SwitchTest2 {
+    public static void main(String[] args) throws Exception {
+        WispEngine.dispatch(() -> {
+            for (int i = 0; i < 9999999; i++) {
+                try {
+                    Thread.sleep(100);
+                    System.out.println(i + ": " + SharedSecrets.getJavaLangAccess().currentThread0());
+                } catch (InterruptedException e) {
+                    e.printStackTrace();
+                }
+            }
+        });
+        System.out.println("Wisp2SwitchTest.main");
+        Field f = Class.forName("com.alibaba.wisp.engine.WispConfiguration").getDeclaredField("WISP_VERSION");
+        f.setAccessible(true);
+        int version = f.getInt(null);
+        assertTrue(version == 2, "Wisp Version isn't Wisp2");
+
+        boolean isEnabled;
+        f = Class.forName("com.alibaba.wisp.engine.WispConfiguration").getDeclaredField("TRANSPARENT_WISP_SWITCH");
+        f.setAccessible(true);
+        isEnabled = f.getBoolean(null);
+        assertTrue(isEnabled == true, "The property com.alibaba.wisp.transparentWispSwitch isn't enabled");
+
+        f = Class.forName("com.alibaba.wisp.engine.WispConfiguration").getDeclaredField("ENABLE_THREAD_AS_WISP");
+        f.setAccessible(true);
+        isEnabled = f.getBoolean(null);
+        assertTrue(isEnabled == true, "The property com.alibaba.wisp.enableThreadAsWisp isn't enabled");
+
+        f = Class.forName("com.alibaba.wisp.engine.WispConfiguration").getDeclaredField("ALL_THREAD_AS_WISP");
+        f.setAccessible(true);
+        isEnabled = f.getBoolean(null);
+        assertTrue(isEnabled == false, "The property com.alibaba.wisp.allThreadAsWisp isn't disabled");
+
+        f = Class.forName("com.alibaba.wisp.engine.WispConfiguration").getDeclaredField("ENABLE_HANDOFF");
+        f.setAccessible(true);
+        isEnabled = f.getBoolean(null);
+        assertTrue(isEnabled == false, "The property com.alibaba.wisp.enableHandOff is enabled");
+
+        Thread.sleep(1000);
+    }
+}
diff --git a/test/hotspot/jtreg/runtime/coroutine/WispClinitTest.java b/test/hotspot/jtreg/runtime/coroutine/WispClinitTest.java
new file mode 100644
index 00000000000..73dd4edd445
--- /dev/null
+++ b/test/hotspot/jtreg/runtime/coroutine/WispClinitTest.java
@@ -0,0 +1,35 @@
+/*
+ * @test
+ * @library /test/lib
+ * @summary test clinit wait in coroutine
+ * @run main/othervm -XX:-UseBiasedLocking -XX:+EnableCoroutine -XX:+UseWispMonitor -Dcom.alibaba.wisp.transparentWispSwitch=true WispClinitTest
+ */
+
+
+import com.alibaba.wisp.engine.WispEngine;
+import static jdk.test.lib.Asserts.assertTrue;
+
+public class WispClinitTest {
+    public static void main(String[] args) {
+        WispEngine.dispatch(C::ensureInit);
+        assertTrue(C.initDone);
+    }
+}
+
+class C {
+    static boolean initDone;
+
+    static {
+        try {
+            Thread.sleep(1000);
+            initDone = true;
+        } catch (InterruptedException e) {
+            e.printStackTrace();
+        }
+    }
+
+    static void ensureInit() {
+    }
+}
+
+
diff --git a/test/hotspot/jtreg/runtime/coroutine/WispEmitNewGuardTest.java b/test/hotspot/jtreg/runtime/coroutine/WispEmitNewGuardTest.java
new file mode 100644
index 00000000000..478e6c72fe1
--- /dev/null
+++ b/test/hotspot/jtreg/runtime/coroutine/WispEmitNewGuardTest.java
@@ -0,0 +1,32 @@
+/*
+ * @test
+ * @library /testlibrary
+ * @summary test emit_guard_for_new in C2 will add control for load
+ * @run main/othervm -Xcomp -XX:-TieredCompilation -Xbatch -XX:CompileOnly=WispEmitNewGuardTest.testMethod -XX:+PrintCompilation -XX:+EnableCoroutine -XX:+UseWispMonitor -Dcom.alibaba.wisp.transparentWispSwitch=true WispEmitNewGuardTest
+ */
+
+
+import com.alibaba.wisp.engine.WispEngine;
+
+public class WispEmitNewGuardTest {
+    static {
+        System.out.println("================ static initialize ================");
+        testMethod();
+        System.out.println("================ static initialize done ================");
+    }
+
+    public static void main(String[] args) throws Exception{
+    }
+
+    public static int testMethod() {
+        WispEmitNewGuardTest x = new WispEmitNewGuardTest(42);
+        return x.value();
+    }
+
+    private int value;
+    WispEmitNewGuardTest(int value) {
+        this.value = value;
+    }
+
+    int value() { return this.value; }
+}
diff --git a/test/hotspot/jtreg/runtime/coroutine/c1AssertFailTest.sh b/test/hotspot/jtreg/runtime/coroutine/c1AssertFailTest.sh
new file mode 100644
index 00000000000..721eda72ae5
--- /dev/null
+++ b/test/hotspot/jtreg/runtime/coroutine/c1AssertFailTest.sh
@@ -0,0 +1,18 @@
+#!/bin/sh
+
+## @test
+##
+## @summary test c1 assertion failure when UseDirectUnpark is enabled (please run it in slowdebug ver.)
+## @run shell c1AssertFailTest.sh
+
+
+${TESTJAVA}/bin/java -Xcomp -XX:-UseBiasedLocking -XX:+EnableCoroutine -XX:+UseWispMonitor -Dcom.alibaba.transparentAsync=true &
+
+PID=$!
+
+sleep 2
+ls -d /proc/$PID || exit 1 # process exited, fail
+
+kill -KILL $PID
+
+exit 0
diff --git a/test/hotspot/jtreg/runtime/coroutine/jniDetachThreadHoldingMonitorTest.c b/test/hotspot/jtreg/runtime/coroutine/jniDetachThreadHoldingMonitorTest.c
new file mode 100644
index 00000000000..db79bd70751
--- /dev/null
+++ b/test/hotspot/jtreg/runtime/coroutine/jniDetachThreadHoldingMonitorTest.c
@@ -0,0 +1,76 @@
+#include 
+#include 
+#include 
+#include 
+#include 
+
+
+JavaVM *jvm;       /* denotes a Java VM */
+
+jobject lock;
+volatile int fooGotLock;
+
+int
+main()
+{
+    JNIEnv *env;
+    JavaVMInitArgs vm_args; /* JDK/JRE 6 VM initialization arguments */
+    JavaVMOption options[4];
+    options[0].optionString = "-XX:+EnableCoroutine";
+    options[1].optionString = "-XX:-UseBiasedLocking";
+    options[2].optionString = "-Dcom.alibaba.transparentAsync=true";
+    options[3].optionString = "-XX:+UseWispMonitor";
+    vm_args.version = JNI_VERSION_1_6;
+    vm_args.nOptions = 4;
+    vm_args.options = options;
+    vm_args.ignoreUnrecognized = false;
+    /* load and initialize a Java VM, return a JNI interface
+     * pointer in env */
+    if (JNI_CreateJavaVM(&jvm, (void**)&env, &vm_args) != JNI_OK) {
+        exit(-1);
+    }
+
+    jclass cls = env->FindClass("java/nio/channels/spi/SelectorProvider");
+    printf("class = %p\n", cls);
+    jfieldID fid = env->GetStaticFieldID(cls, "lock", "Ljava/lang/Object;");
+    printf("fid = %p\n", fid);
+    lock = env->GetStaticObjectField(cls, fid);
+    printf("lock = %p\n", lock);
+
+    pthread_t tid;
+    void *foo(void *arg);
+    pthread_create(&tid, NULL, foo, NULL);
+
+    while (!fooGotLock)
+        ; // wait
+
+    if (env->MonitorEnter(lock) != JNI_OK) {
+        exit(-1);
+    }
+
+    return jvm->DestroyJavaVM() == JNI_OK ? 0 : -1;
+}
+
+
+void *foo(void *arg)
+{
+
+    JNIEnv *env;
+    if (jvm->AttachCurrentThread((void**)&env, NULL) != JNI_OK) {
+        exit(-1);
+    }
+
+    if (env->MonitorEnter(lock) != JNI_OK) {
+        exit(-1);
+    }
+
+    fooGotLock = 1;
+
+    usleep(100 * 1000); // 100 ms
+
+    if (jvm->DetachCurrentThread() != JNI_OK) { // unpark main thread
+        exit(-1);
+    }
+
+    return NULL;
+}
diff --git a/test/hotspot/jtreg/runtime/coroutine/jniDetachThreadHoldingMonitorTest.sh b/test/hotspot/jtreg/runtime/coroutine/jniDetachThreadHoldingMonitorTest.sh
new file mode 100644
index 00000000000..0f266fcc012
--- /dev/null
+++ b/test/hotspot/jtreg/runtime/coroutine/jniDetachThreadHoldingMonitorTest.sh
@@ -0,0 +1,18 @@
+#!/bin/sh
+
+## @test
+##
+## @summary test DetachCurrentThread unpark
+## @run shell jniDetachThreadHoldingMonitorTest.sh
+##
+
+
+export LD_LIBRARY_PATH=.:${COMPILEJAVA}/lib/server:/usr/lib:$LD_LIBRARY_PATH
+
+g++ -DLINUX -o jniDetachThreadHoldingMonitorTest \
+    -I${COMPILEJAVA}/include -I${COMPILEJAVA}/include/linux \
+    -L${COMPILEJAVA}/lib/server \
+    -ljvm -lpthread ${TESTSRC}/jniDetachThreadHoldingMonitorTest.c
+
+./jniDetachThreadHoldingMonitorTest
+exit $?
diff --git a/test/hotspot/jtreg/runtime/coroutine/jniMonitorExitTest.c b/test/hotspot/jtreg/runtime/coroutine/jniMonitorExitTest.c
new file mode 100644
index 00000000000..5d2c41dbf0e
--- /dev/null
+++ b/test/hotspot/jtreg/runtime/coroutine/jniMonitorExitTest.c
@@ -0,0 +1,53 @@
+#include 
+#include 
+#include 
+#include 
+#include 
+
+
+JavaVM *jvm;       /* denotes a Java VM */
+
+jobject lock;
+volatile int fooGotLock;
+
+int
+main()
+{
+    JNIEnv *env;
+    JavaVMInitArgs vm_args; /* JDK/JRE 6 VM initialization arguments */
+    JavaVMOption options[4];
+    options[0].optionString = "-XX:+EnableCoroutine";
+    options[1].optionString = "-XX:-UseBiasedLocking";
+    options[2].optionString = "-Dcom.alibaba.transparentAsync=true";
+    options[3].optionString = "-XX:+UseWispMonitor";
+    vm_args.version = JNI_VERSION_1_6;
+    vm_args.nOptions = 4;
+    vm_args.options = options;
+    vm_args.ignoreUnrecognized = false;
+    /* load and initialize a Java VM, return a JNI interface
+     * pointer in env */
+    if (JNI_CreateJavaVM(&jvm, (void**)&env, &vm_args) != JNI_OK) {
+        exit(-1);
+    }
+
+    jclass cls = env->FindClass("java/nio/channels/spi/SelectorProvider");
+    printf("class = %p\n", cls);
+    jfieldID fid = env->GetStaticFieldID(cls, "lock", "Ljava/lang/Object;");
+    printf("fid = %p\n", fid);
+    lock = env->GetStaticObjectField(cls, fid);
+    printf("lock = %p\n", lock);
+
+    if (env->MonitorEnter(lock) != JNI_OK) {
+        exit(-1);
+    }
+
+    if (env->MonitorExit(lock) != JNI_OK) {
+        if (env->ExceptionOccurred()) {
+          env->ExceptionDescribe(); // print the stack trace          
+        }
+        exit(-1);
+    }
+
+    return jvm->DestroyJavaVM() == JNI_OK ? 0 : -1;
+}
+
diff --git a/test/hotspot/jtreg/runtime/coroutine/jniMonitorExitTest.sh b/test/hotspot/jtreg/runtime/coroutine/jniMonitorExitTest.sh
new file mode 100644
index 00000000000..ede41f4360c
--- /dev/null
+++ b/test/hotspot/jtreg/runtime/coroutine/jniMonitorExitTest.sh
@@ -0,0 +1,19 @@
+#!/bin/sh
+
+## @test
+##
+## @summary test jni MonitorExit
+## @run shell jniMonitorExitTest.sh
+##
+
+
+export LD_LIBRARY_PATH=.:${COMPILEJAVA}/lib/server:/usr/lib:$LD_LIBRARY_PATH
+echo ${COMPILEJAVA}
+echo $LD_LIBRARY_PATH
+g++ -DLINUX -o jniMonitorExitTest \
+    -I${COMPILEJAVA}/include -I${COMPILEJAVA}/include/linux \
+    -L${COMPILEJAVA}/lib/server \
+    -ljvm -lpthread ${TESTSRC}/jniMonitorExitTest.c
+
+./jniMonitorExitTest
+exit $?
diff --git a/test/hotspot/jtreg/runtime/coroutine/jvmtiWispMonitorTest.sh b/test/hotspot/jtreg/runtime/coroutine/jvmtiWispMonitorTest.sh
new file mode 100644
index 00000000000..26ec971d046
--- /dev/null
+++ b/test/hotspot/jtreg/runtime/coroutine/jvmtiWispMonitorTest.sh
@@ -0,0 +1,19 @@
+#!/bin/sh
+
+## @test
+##
+## @summary test jvmti and wispMonitor could work together
+## @run shell jvmtiWispMonitorTest.sh
+
+
+${TESTJAVA}/bin/java -agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=localhost:5005 -XX:+EnableCoroutine -XX:+UseWispMonitor &
+
+PID=$!
+
+sleep 2
+ls -d /proc/$PID || exit 1 # process exited, fail
+
+kill -KILL $PID
+
+exit 0
+
diff --git a/test/hotspot/jtreg/runtime/coroutine/logCompilationTest.sh b/test/hotspot/jtreg/runtime/coroutine/logCompilationTest.sh
new file mode 100644
index 00000000000..76cc4512aee
--- /dev/null
+++ b/test/hotspot/jtreg/runtime/coroutine/logCompilationTest.sh
@@ -0,0 +1,33 @@
+#!/bin/sh
+
+#
+# @test
+# @library /testlibrary
+# @compile SimpleWispTest.java
+#
+# @summary test coroutine and -XX:+LogCompilation could work together
+# @run shell logCompilationTest.sh
+#
+
+OS=`uname -s`
+case "$OS" in
+  AIX | Darwin | Linux | SunOS )
+    FS="/"
+    ;;
+  Windows_* )
+    FS="\\"
+    ;;
+  CYGWIN_* )
+    FS="/"
+    ;;
+  * )
+    echo "Unrecognized system!"
+    exit 1;
+    ;;
+esac
+
+JAVA=${TESTJAVA}${FS}bin${FS}java
+
+${JAVA} -XX:+UnlockDiagnosticVMOptions -XX:+EnableCoroutine -XX:+UseWispMonitor -Dcom.alibaba.transparentAsync=true -XX:+LogCompilation -Xcomp -cp ${TESTCLASSES} SimpleWispTest
+
+exit $?
\ No newline at end of file
diff --git a/test/hotspot/jtreg/runtime/coroutine/premainWithWispMonitorTest.sh b/test/hotspot/jtreg/runtime/coroutine/premainWithWispMonitorTest.sh
new file mode 100644
index 00000000000..c05673e1917
--- /dev/null
+++ b/test/hotspot/jtreg/runtime/coroutine/premainWithWispMonitorTest.sh
@@ -0,0 +1,36 @@
+#!/bin/sh
+# see PremainWithWispMonitorTest.java
+
+AGENT=PremainWithWispMonitorTest
+
+if [ "${TESTSRC}" = "" ]
+then
+  echo "TESTSRC not set.  Test cannot execute.  Failed."
+  exit 1
+fi
+echo "TESTSRC=${TESTSRC}"
+
+if [ "${TESTJAVA}" = "" ]
+then
+  echo "TESTJAVA not set.  Test cannot execute.  Failed."
+  exit 1
+fi
+echo "TESTJAVA=${TESTJAVA}"
+
+
+cp ${TESTSRC}/${AGENT}.java .
+${TESTJAVA}/bin/javac ${TESTJAVACOPTS} ${TESTTOOLVMOPTS} PremainWithWispMonitorTest.java
+
+echo "Manifest-Version: 1.0"    >  ${AGENT}.mf
+echo Premain-Class: ${AGENT} >> ${AGENT}.mf
+shift
+while [ $# != 0 ] ; do
+  echo $1 >> ${AGENT}.mf
+  shift
+done
+
+${TESTJAVA}/bin/jar ${TESTTOOLVMOPTS} cvfm ${AGENT}.jar ${AGENT}.mf ${AGENT}*.class
+
+${TESTJAVA}/bin/java -XX:+EnableCoroutine -Dcom.alibaba.transparentAsync=true -XX:+UseWispMonitor \
+        -javaagent:PremainWithWispMonitorTest.jar PremainWithWispMonitorTest
+
diff --git a/test/jdk/com/alibaba/wisp/ConfigurationCompatibilityCheckTest.java b/test/jdk/com/alibaba/wisp/ConfigurationCompatibilityCheckTest.java
new file mode 100644
index 00000000000..847a26149a2
--- /dev/null
+++ b/test/jdk/com/alibaba/wisp/ConfigurationCompatibilityCheckTest.java
@@ -0,0 +1,33 @@
+/*
+ * @test
+ * @summary Test the config compatibility in different wisp version
+ * @library /test/lib
+ * @run main ConfigurationCompatibilityCheckTest
+ */
+import java.util.ArrayList;
+import java.util.Arrays;
+import jdk.test.lib.process.OutputAnalyzer;
+import jdk.test.lib.process.ProcessTools;
+
+public class ConfigurationCompatibilityCheckTest {
+    public static void main(String[] args) throws Exception {
+        incompatibility("-Dcom.alibaba.wisp.version=65536");
+        incompatibility("-Dcom.alibaba.wisp.enableThreadAsWisp=true");
+        incompatibility("-Dcom.alibaba.wisp.enableThreadAsWisp=true", "-Dcom.alibaba.wisp.transparentWispSwitch=false");
+        incompatibility("-Dcom.alibaba.wisp.version=2", "-Dcom.alibaba.globalPoller=false");
+        incompatibility("-Dcom.alibaba.wisp.allThreadAsWisp=true");
+        incompatibility("-Dcom.alibaba.wisp.allThreadAsWisp=true", "-Dcom.alibaba.wisp.version=1");
+        incompatibility("-Dcom.alibaba.wisp.allThreadAsWisp=true", "-Dcom.alibaba.wisp.enableThreadAsWisp=false");
+    }
+
+
+    private static void incompatibility(String... args) throws Exception {
+        ArrayList list = new ArrayList<>();
+        list.add("-XX:+EnableCoroutine");
+        list.addAll(Arrays.asList(args));
+        list.add("-version");
+        ProcessBuilder pb = ProcessTools.createJavaProcessBuilder(list.toArray(new String[0]));
+        OutputAnalyzer output = new OutputAnalyzer(pb.start());
+        output.shouldContain("IllegalArgumentException");
+    }
+}
diff --git a/test/jdk/com/alibaba/wisp/CoroutineTest.java b/test/jdk/com/alibaba/wisp/CoroutineTest.java
new file mode 100644
index 00000000000..cd3ee800e9d
--- /dev/null
+++ b/test/jdk/com/alibaba/wisp/CoroutineTest.java
@@ -0,0 +1,55 @@
+/*
+ * @test
+ * @summary Test low level coroutine implement
+ * @run main/othervm -XX:+EnableCoroutine CoroutineTest
+ * @run main/othervm -XX:+EnableCoroutine -Dcom.alibaba.wisp.version=2 CoroutineTest
+*/
+
+import java.dyn.Coroutine;
+import java.lang.reflect.Constructor;
+import java.lang.reflect.Method;
+
+/**
+ * test low level coroutine implement
+ * 2 coroutine switch to each other and see the sequence
+ */
+public class CoroutineTest {
+    static int i = 0;
+    static Coroutine co1, co2;
+
+    public static void main(String[] args) {
+        try {
+
+            co1 = new Coroutine(() -> {
+                try {
+                    if (i++ != 0)
+                        throw new RuntimeException("co1 wrong sequence, expect 0");
+                    Coroutine.yieldTo(co2);
+                    if (i++ != 2)
+                        throw new RuntimeException("co1 wrong sequence, expect 2");
+                    Coroutine.yieldTo(co2);
+                } catch (Exception e) {
+                    throw new RuntimeException(e);
+                }
+            });
+            co2 = new Coroutine(() -> {
+
+                try {
+                    if (i++ != 1)
+                        throw new RuntimeException("co2 wrong sequence, expect 1");
+                    Coroutine.yieldTo(co1);
+                    if (i++ != 3)
+                        throw new RuntimeException("co2 wrong sequence, expect 3");
+                } catch (Exception e) {
+                    throw new RuntimeException(e);
+                }
+            });
+
+            Coroutine.yieldTo(co1);
+
+
+        } catch (Exception e) {
+            throw new RuntimeException();
+        }
+    }
+}
diff --git a/test/jdk/com/alibaba/wisp/ExecutionTest.java b/test/jdk/com/alibaba/wisp/ExecutionTest.java
new file mode 100644
index 00000000000..35742fdf676
--- /dev/null
+++ b/test/jdk/com/alibaba/wisp/ExecutionTest.java
@@ -0,0 +1,124 @@
+/*
+ * @test
+ * @summary Test WispEngine's multi-task schedule
+ * @modules java.base/jdk.internal.misc
+ * @run main/othervm -XX:+EnableCoroutine -Dcom.alibaba.wisp.transparentWispSwitch=true ExecutionTest
+ * @run main/othervm -XX:+EnableCoroutine -Dcom.alibaba.wisp.transparentWispSwitch=true -Dcom.alibaba.wisp.version=2 ExecutionTest
+*/
+
+
+import com.alibaba.wisp.engine.WispEngine;
+import com.alibaba.wisp.engine.WispTask;
+import jdk.internal.access.SharedSecrets;
+
+import java.util.concurrent.Callable;
+import java.util.concurrent.FutureTask;
+
+/**
+ * test the coroutine execute engine
+ *
+ */
+public class ExecutionTest {
+
+    static int finishCnt = 0;
+    public static void main(String[] args) {
+        Callable handler = () -> {
+            /**
+             * transform from:
+             * if ((nodeA() + nodeB()).equals("A:done\nB1:done\nB2:done\n"))
+             *     throw new Error("result error");
+             */
+            FutureTask futureA = new FutureTask<>(ExecutionTest::nodeA);
+            FutureTask futureB = new FutureTask<>(ExecutionTest::nodeB);
+            WispEngine.dispatch(futureA);
+            WispEngine.dispatch(futureB);
+            String result;
+            try {
+                result = futureA.get() + futureB.get();
+            } catch (Exception e) {
+                throw new Error("exception during task execution");
+            }
+
+            if (!result.equals("A:done\nB1:done\nB2:done\n"))
+                throw new Error("result error");
+
+            finishCnt++;
+            return null;
+        };
+
+        /**
+         * 
+         * WispEngine.local().createTask(() -> {
+         *     while (client = accept()) {
+         *         WispEngine.local().createTask(()->handler(client), "client handler");
+         *     }
+         * }, "accepter");
+         *
+         * a web server can using a accepter {@link WispTask}  create handler for every request
+         * we don't have request, create 3 handler manually
+         *
+         * every handler is a tree root
+         *         handler         handler        handler
+         *         /    \           /   \          /   \
+         *        A     B          A    B         A    B
+         *             / \             / \            / \
+         *            B1 B2           B1 B2          B1 B2
+         * 

+         *
+         * look into a particular tree:
+         * B1 and B2 is created when nodeB is executed after nodeA blocked
+         * business code drive the node create ..
+         *
+         * then B1 block; B2 executed and block
+         * then the engine block on pump about 100ms..
+         *
+         *
+         * 3 tree execute concurrently
+         *
+         */
+        for (int i = 0; i < 3; i++) {
+            WispEngine.dispatch(() -> {
+                try {
+                    handler.call();
+                } catch (Exception e) {
+                    e.printStackTrace();
+                }
+            });
+        }
+
+        WispEngine.dispatch(() -> {
+            SharedSecrets.getWispEngineAccess().sleep(200);
+            // the 3 tree should finish after 100ms+, but the switch need warm up, give more time..
+            if (finishCnt != 3) throw new Error("not finished");
+        });
+        SharedSecrets.getWispEngineAccess().eventLoop(); // run the engine
+    }
+
+    static String nodeA() {
+        // node A could also be a function call after B created
+        return slowReq("A");
+    }
+
+    static String nodeB() {
+        /*
+          transform from:
+          return slowReq("B1") + slowReq("B2");
+         */
+        FutureTask future1 = new FutureTask<>(() -> slowReq("B1"));
+        FutureTask future2 = new FutureTask<>(() -> slowReq("B2"));
+        WispEngine.dispatch(future1);
+        WispEngine.dispatch(future2);
+
+        try {
+            return future1.get() + future2.get();
+        } catch (Exception e) {
+            throw new Error("exception during task execution");
+        }
+    }
+
+    // mimic an IO call
+    static String slowReq(String arg) {
+        SharedSecrets.getWispEngineAccess().sleep(100); // only block current coroutine
+        return arg + ":done\n";
+    }
+}
diff --git a/test/jdk/com/alibaba/wisp/IoTest.java b/test/jdk/com/alibaba/wisp/IoTest.java
new file mode 100644
index 00000000000..5fa35f0dff6
--- /dev/null
+++ b/test/jdk/com/alibaba/wisp/IoTest.java
@@ -0,0 +1,85 @@
+/*
+ * @test
+ * @summary Test Wisp engine's NIO support
+ * @modules java.base/jdk.internal.misc
+ * @run main/othervm -XX:+EnableCoroutine IoTest
+ * @run main/othervm -XX:+EnableCoroutine -Dcom.alibaba.wisp.version=2 IoTest
+*/
+
+
+
+import jdk.internal.access.SharedSecrets;
+
+import java.io.IOException;
+import java.net.InetAddress;
+import java.net.InetSocketAddress;
+import java.nio.ByteBuffer;
+import java.nio.channels.SelectionKey;
+import java.nio.channels.SocketChannel;
+
+public class IoTest {
+    public static void main(String[] args) {
+        String result = http();
+
+        System.out.println(result);
+
+        if (!result.startsWith("HTTP") ||
+                !result.contains("Content-") ||
+                !result.contains("Server:"))
+            throw new Error("protocol error");
+    }
+
+    static String http() {
+        String host = "www.example.com";
+
+        try {
+            SocketChannel ch = SocketChannel.open();
+            ch.configureBlocking(false);
+
+            connect(ch, InetAddress.getByName(host).getHostAddress());
+            ByteBuffer bb = ByteBuffer.allocate(1000);
+            String request = "HEAD / HTTP/1.0\r\nHost:" + host + "\r\n\r\n";
+            bb.put(request.getBytes());
+            bb.flip();
+            write(ch, bb);
+            bb.clear();
+            read(ch, bb);
+
+            return new String(bb.array(), 0, bb.remaining());
+
+        } catch (IOException e) {
+            throw new Error(e);
+        }
+    }
+
+    static int read(SocketChannel ch, ByteBuffer bb) throws IOException {
+        int n, retry = 0;
+        while ((n = ch.read(bb)) == 0) {
+            if (retry++ > 3)
+                throw new Error("busy loop"); // make sure we're not in a busy loop
+            SharedSecrets.getWispEngineAccess().registerEvent(ch, SelectionKey.OP_READ);
+            SharedSecrets.getWispEngineAccess().park(-1);
+        }
+        return n;
+    }
+
+    static int write(SocketChannel ch, ByteBuffer bb) throws IOException {
+        int n, retry = 0;
+        while ((n = ch.write(bb)) == 0) {
+            if (retry++ > 3)    throw new Error("busy loop");
+            SharedSecrets.getWispEngineAccess().registerEvent(ch, SelectionKey.OP_WRITE);
+            SharedSecrets.getWispEngineAccess().park(-1);
+        }
+        return n;
+    }
+
+    static void connect(SocketChannel ch, String ip) throws IOException {
+        ch.connect(new InetSocketAddress(ip, 80));
+        int retry = 0;
+        while (!ch.finishConnect()) {
+            if (retry++ > 3)    throw new Error("busy loop");
+            SharedSecrets.getWispEngineAccess().registerEvent(ch, SelectionKey.OP_CONNECT);
+            SharedSecrets.getWispEngineAccess().park(-1);
+        }
+    }
+}
diff --git a/test/jdk/com/alibaba/wisp/ParkTest.java b/test/jdk/com/alibaba/wisp/ParkTest.java
new file mode 100644
index 00000000000..ef2e07b2a78
--- /dev/null
+++ b/test/jdk/com/alibaba/wisp/ParkTest.java
@@ -0,0 +1,65 @@
+/*
+ * @test
+ * @summary Test Wisp engine park / unpark
+ * @modules java.base/jdk.internal.misc
+ * @run main/othervm -XX:+EnableCoroutine ParkTest
+ * @run main/othervm -XX:+EnableCoroutine -Dcom.alibaba.wisp.version=2 ParkTest
+*/
+
+
+import com.alibaba.wisp.engine.WispEngine;
+import com.alibaba.wisp.engine.WispTask;
+import jdk.internal.access.SharedSecrets;
+import jdk.internal.access.WispEngineAccess;
+
+import java.util.concurrent.TimeUnit;
+
+public class ParkTest {
+    public static void main(String[] args) {
+        WispEngineAccess access = SharedSecrets.getWispEngineAccess();
+
+        WispTask[] task = new WispTask[1];
+
+        WispEngine.dispatch(() -> {
+            task[0] = access.getCurrentTask();
+            long start, diff;
+
+            start = System.currentTimeMillis();
+            access.park(0);
+
+            diff = System.currentTimeMillis() - start;
+            if (diff < 200 || diff > 220)
+                throw new Error("error test unpark by other thread");
+
+
+
+            start = start + diff;
+            access.park(TimeUnit.MILLISECONDS.toNanos(200));
+            diff = System.currentTimeMillis() - start;
+
+            if (diff < 200 || diff > 220)
+                throw new Error("error test timed park");
+
+
+
+            start = start + diff;
+            access.unpark(access.getCurrentTask());
+            access.park(0);
+            diff = System.currentTimeMillis() - start;
+            if (diff > 20)
+                throw new Error("error test permitted park");
+
+        });
+
+        Thread unparkThread = new Thread() {
+            @Override
+            public void run() {
+                access.sleep(200);
+                access.unpark(task[0]);
+            }
+        };
+        unparkThread.start();
+
+        access.eventLoop();
+    }
+}
diff --git a/test/jdk/com/alibaba/wisp/SelectorLazyCreateTest.java b/test/jdk/com/alibaba/wisp/SelectorLazyCreateTest.java
new file mode 100644
index 00000000000..033ac0a9066
--- /dev/null
+++ b/test/jdk/com/alibaba/wisp/SelectorLazyCreateTest.java
@@ -0,0 +1,32 @@
+/*
+ * @test
+ * @library /lib/testlibrary
+ * @summary Test for engine.selector lazy created
+ * @modules java.base/com.alibaba.wisp.engine:+open
+ * @run main/othervm -XX:+EnableCoroutine -Dcom.alibaba.wisp.transparentWispSwitch=true -Dcom.alibaba.globalPoller=false SelectorLazyCreateTest
+ */
+
+import com.alibaba.wisp.engine.WispEngine;
+
+import java.lang.reflect.Field;
+import java.net.Socket;
+
+import static jdk.testlibrary.Asserts.assertNotNull;
+import static jdk.testlibrary.Asserts.assertNull;
+
+public class SelectorLazyCreateTest {
+
+    public static void main(String[] args) throws Exception {
+        if (!Class.forName("com.alibaba.wisp.engine.ScheduledWispEngine").isInstance(WispEngine.current())) {
+          return;
+        }
+        Field selField = Class.forName("com.alibaba.wisp.engine.ScheduledWispEngine").getDeclaredField("selector");
+        selField.setAccessible(true);
+
+        assertNull(selField.get(WispEngine.current()));
+
+        new Socket("wwww.taobao.com", 80);
+
+        assertNotNull(selField.get(WispEngine.current()));
+    }
+}
diff --git a/test/jdk/com/alibaba/wisp/boot/UnsafeDependencyBugTest.java b/test/jdk/com/alibaba/wisp/boot/UnsafeDependencyBugTest.java
new file mode 100644
index 00000000000..5fa9036da0e
--- /dev/null
+++ b/test/jdk/com/alibaba/wisp/boot/UnsafeDependencyBugTest.java
@@ -0,0 +1,37 @@
+/*
+ * @test
+ * @library /lib/testlibrary
+ * @summary test bug fix of SharedSecrets and Unsafe class initializer circular dependency
+ * @run main UnsafeDependencyBugTest 10
+ */
+
+
+import java.io.File;
+import java.util.concurrent.TimeUnit;
+
+import static jdk.testlibrary.Asserts.assertTrue;
+
+/**
+ * We need thousand times to reproduce the DEADLOCK. Don't spend too much time here..
+ * We already add svt test ajdk_svt/wispTest to ensure the DEADLOCK already solved.
+ */
+public class UnsafeDependencyBugTest {
+
+    public static void main(String[] args) throws Exception {
+        long start = System.currentTimeMillis();
+        int i;
+        for (i = 0; System.currentTimeMillis() - start < Integer.valueOf(args[0]) * 1000; i++) {
+            runLauncherWithWisp();
+        }
+        System.out.println("tested " +  i + " times");
+    }
+
+    private static void runLauncherWithWisp() throws Exception {
+        Process p = new ProcessBuilder(System.getProperty("java.home") + "/bin/java", "-XX:+EnableCoroutine")
+                .redirectErrorStream(true)
+                .redirectOutput(new File("/dev/null"))
+                .start();
+
+        assertTrue(p.waitFor(2, TimeUnit.SECONDS));
+    }
+}
diff --git a/test/jdk/com/alibaba/wisp/bug/CancelTimerAndSleepTest.java b/test/jdk/com/alibaba/wisp/bug/CancelTimerAndSleepTest.java
new file mode 100644
index 00000000000..f6a79a70807
--- /dev/null
+++ b/test/jdk/com/alibaba/wisp/bug/CancelTimerAndSleepTest.java
@@ -0,0 +1,28 @@
+/*
+ * @test
+ * @summary Test sleep
+ * @modules java.base/jdk.internal.misc
+ * @run main/othervm -XX:+EnableCoroutine -Dcom.alibaba.wisp.transparentWispSwitch=true CancelTimerAndSleepTest
+ * @run main/othervm -XX:+EnableCoroutine -Dcom.alibaba.wisp.transparentWispSwitch=true -Dcom.alibaba.wisp.version=2 CancelTimerAndSleepTest
+*/
+
+import jdk.internal.access.SharedSecrets;
+import jdk.internal.access.WispEngineAccess;
+
+import java.util.concurrent.TimeUnit;
+
+public class CancelTimerAndSleepTest {
+
+    public static void main(String[] args) throws Exception {
+        WispEngineAccess access = SharedSecrets.getWispEngineAccess();
+
+        access.addTimer(System.nanoTime() + TimeUnit.MILLISECONDS.toNanos(10));
+        access.cancelTimer();
+
+        long now = System.currentTimeMillis();
+        Thread.sleep(200);
+        long elapsed = System.currentTimeMillis() - now;
+        if (Math.abs(elapsed - 200) > 10)
+            throw new Error("elapsed = " + elapsed);
+    }
+}
diff --git a/test/jdk/com/alibaba/wisp/bug/ClearEventTest.java b/test/jdk/com/alibaba/wisp/bug/ClearEventTest.java
new file mode 100644
index 00000000000..da37806ddac
--- /dev/null
+++ b/test/jdk/com/alibaba/wisp/bug/ClearEventTest.java
@@ -0,0 +1,84 @@
+/*
+ * @test
+ * @summary Explain the fix of T11748781
+ * @modules java.base/jdk.internal.misc
+ * @modules java.base/com.alibaba.wisp.engine:+open
+ * @run main/othervm -XX:+EnableCoroutine -Dcom.alibaba.wisp.transparentWispSwitch=true ClearEventTest
+ * @run main/othervm -XX:+EnableCoroutine -Dcom.alibaba.wisp.transparentWispSwitch=true -Dcom.alibaba.wisp.version=2 ClearEventTest
+*/
+
+import com.alibaba.wisp.engine.WispEngine;
+import jdk.internal.access.SharedSecrets;
+import jdk.internal.access.WispEngineAccess;
+
+import java.io.IOException;
+import java.lang.reflect.Field;
+import java.net.Socket;
+import java.nio.channels.CancelledKeyException;
+import java.nio.channels.SelectionKey;
+import java.nio.channels.Selector;
+import java.nio.channels.SocketChannel;
+import java.util.concurrent.CountDownLatch;
+
+public class ClearEventTest {
+    private static WispEngineAccess access = SharedSecrets.getWispEngineAccess();
+
+    private static CountDownLatch threadRun = new CountDownLatch(1);
+    private static CountDownLatch closed = new CountDownLatch(1);
+
+
+    public static void main(String[] args) throws Exception {
+        Socket so = new Socket("www.example.com", 80);
+        so.getOutputStream().write("NOP\n\r\n\r".getBytes());
+        // now server returns the data..
+        // so is readable
+        // current task is interested in read event.
+        SocketChannel ch = so.getChannel();
+        access.registerEvent(ch, SelectionKey.OP_READ);
+        access.park(-1);
+
+        if (!Class.forName("com.alibaba.wisp.engine.ScheduledWispEngine").isInstance(WispEngine.current())) {
+            return;
+        }
+        Field f = Class.forName("com.alibaba.wisp.engine.ScheduledWispEngine").getDeclaredField("selector");
+        f.setAccessible(true);
+        Selector sel = (Selector) f.get(WispEngine.current());
+
+        new Thread(() -> {
+            try {
+                threadRun.await();
+                so.close();
+            } catch (IOException | InterruptedException e) {
+                e.printStackTrace();
+            }
+            closed.countDown();
+        }).start();
+
+        while (true) {
+            try {
+                cleanEvent(sel, ch);
+                break;
+            } catch (CancelledKeyException e) {
+                continue;
+            }
+        }
+    }
+
+    private static void cleanEvent(Selector sel, SocketChannel ch) throws InterruptedException {
+        SelectionKey k = ch.keyFor(sel);
+        if (k != null && k.isValid()) {
+            threadRun.countDown();
+            closed.await();
+
+            // even we checked k.isValid()
+            // but if another thread  closed channel here,
+            // the next line will produce CancelledKeyException
+
+            try {
+                k.interestOps(0);
+            } catch (CancelledKeyException e) {
+                // channel already closed, do nothing
+            }
+        }
+    }
+}
diff --git a/test/jdk/com/alibaba/wisp/bug/Id2TaskMapLeakTest.java b/test/jdk/com/alibaba/wisp/bug/Id2TaskMapLeakTest.java
new file mode 100644
index 00000000000..b93a1e8292d
--- /dev/null
+++ b/test/jdk/com/alibaba/wisp/bug/Id2TaskMapLeakTest.java
@@ -0,0 +1,37 @@
+/*
+ * @test
+ * @library /lib/testlibrary
+ * @summary Test for thread WispTask leak
+ * @modules java.base/com.alibaba.wisp.engine:+open
+ * @run main/othervm -XX:+EnableCoroutine -Dcom.alibaba.wisp.transparentWispSwitch=true Id2TaskMapLeakTest
+ * @run main/othervm -XX:+EnableCoroutine -Dcom.alibaba.wisp.transparentWispSwitch=true -Dcom.alibaba.wisp.version=2 Id2TaskMapLeakTest
+*/
+import com.alibaba.wisp.engine.WispEngine;
+import com.alibaba.wisp.engine.WispTask;
+
+import java.lang.reflect.Field;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import static jdk.testlibrary.Asserts.assertEQ;
+
+public class Id2TaskMapLeakTest {
+    public static void main(String[] args) throws Exception {
+        Field f = WispTask.class.getDeclaredField("id2Task");
+        f.setAccessible(true);
+        Map map = (Map) f.get(null);
+
+        int size0 = map.size();
+
+        AtomicInteger sizeHolder = new AtomicInteger();
+
+        new Thread(() -> {
+            WispEngine.current();
+            sizeHolder.set(map.size());
+        }).start();
+        Thread.sleep(20); // ensure thread exit;
+
+        assertEQ(size0 + 1, sizeHolder.get());
+        assertEQ(size0, map.size());
+    }
+}
diff --git a/test/jdk/com/alibaba/wisp/bug/ReleaseWispSocketAndExitTest.java b/test/jdk/com/alibaba/wisp/bug/ReleaseWispSocketAndExitTest.java
new file mode 100644
index 00000000000..9115b2a3385
--- /dev/null
+++ b/test/jdk/com/alibaba/wisp/bug/ReleaseWispSocketAndExitTest.java
@@ -0,0 +1,88 @@
+/*
+ * @test
+ * @summary handle this scenario:
+ *      1. task A fetch a socket S and release it.
+ *      2. task B get the socket S and block on IO.
+ *      3. task A exit and clean S's event, now B waiting forever...
+ * @modules java.base/jdk.internal.misc
+ * @run main/othervm -XX:+EnableCoroutine -Dcom.alibaba.wisp.transparentWispSwitch=true -Dcom.alibaba.wisp.version=1 ReleaseWispSocketAndExitTest
+ * @run main/othervm -XX:+EnableCoroutine -Dcom.alibaba.wisp.transparentWispSwitch=true -Dcom.alibaba.wisp.version=2 ReleaseWispSocketAndExitTest
+*/
+
+
+import com.alibaba.wisp.engine.WispEngine;
+import jdk.internal.access.SharedSecrets;
+
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.net.ServerSocket;
+import java.net.Socket;
+import java.util.concurrent.CountDownLatch;
+
+public class ReleaseWispSocketAndExitTest {
+    static byte buf[] = new byte[1000];
+
+    public static void main(String[] args) throws Exception {
+
+        CountDownLatch latch = new CountDownLatch(1);
+
+        CountDownLatch listen = new CountDownLatch(1);
+
+        WispEngine.dispatch(() -> {
+            try {
+                ServerSocket sso = new ServerSocket(51243);
+                listen.countDown();
+                Socket so;
+                so = sso.accept();
+                InputStream is = so.getInputStream();
+                OutputStream os = so.getOutputStream();
+                int n;
+                while ((n = is.read(buf, 0, 4)) == 4) {
+                    long l = Long.valueOf(new String(buf, 0, n));
+                    System.out.println("l = " + l);
+                    SharedSecrets.getWispEngineAccess().sleep(l);
+                    os.write(buf, 0, n);
+                }
+            } catch (Exception e) {
+                throw new Error();
+            }
+        });
+
+        listen.await();
+        Socket so = new Socket("127.0.0.1", 51243);
+
+        WispEngine.dispatch(() -> {
+            try {
+                InputStream is = so.getInputStream();
+                OutputStream os = so.getOutputStream();
+                os.write("0000".getBytes());
+                System.out.println("======");
+                latch.await();
+                System.out.println("======2");
+            } catch (Exception e) {
+                throw new Error();
+            }
+        });
+
+        WispEngine.dispatch(() -> {
+            try {
+                InputStream is = so.getInputStream();
+                OutputStream os = so.getOutputStream();
+
+                os.write("0100".getBytes());
+                System.out.println("------");
+                latch.countDown();
+                is.read(buf);
+                is.read(buf);// blocked
+                System.out.println("------2");
+
+                so.close();
+            } catch (Exception e) {
+                throw new Error();
+            }
+        });
+
+        SharedSecrets.getWispEngineAccess().eventLoop();
+        System.out.println("passed");
+    }
+}
diff --git a/test/jdk/com/alibaba/wisp/bug/ResetTaskCancelTimerBugTest.java b/test/jdk/com/alibaba/wisp/bug/ResetTaskCancelTimerBugTest.java
new file mode 100644
index 00000000000..34a44805294
--- /dev/null
+++ b/test/jdk/com/alibaba/wisp/bug/ResetTaskCancelTimerBugTest.java
@@ -0,0 +1,49 @@
+/*
+ * @test
+ * @library /lib/testlibrary
+ * @summary test reset task doesn't cancel the current task's timer unexpectedly.
+ * @run main/othervm -XX:+EnableCoroutine -Dcom.alibaba.wisp.transparentWispSwitch=true -Dcom.alibaba.wisp.version=1 ResetTaskCancelTimerBugTest
+ */
+
+import com.alibaba.wisp.engine.WispEngine;
+
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicReference;
+
+import static jdk.testlibrary.Asserts.assertTrue;
+
+
+public class ResetTaskCancelTimerBugTest {
+    public static void main(String[] args) throws Exception {
+        AtomicReference executor = new AtomicReference<>();
+        AtomicBoolean submitDone = new AtomicBoolean();
+        CountDownLatch latch = new CountDownLatch(1);
+
+        new Thread(() -> {
+            executor.set(WispEngine.current());
+            while (!submitDone.get()) {}
+            // 4. go sleep, cause there's a pending external WispTask create operation,
+            // we should create the task fist.
+            // WispEngine.runTaskInternal() -> WispTask.reset() -> WispEngine.cancelTimer()
+            // will remove current task's timer, the sleep() could never be wakened.
+            try {
+                Thread.sleep(500);
+            } catch (InterruptedException e) {
+                e.printStackTrace();
+            }
+            latch.countDown();
+        }).start();
+        while (executor.get() == null) {}
+
+        // 1. create a thread(also created WispEngine A)
+        executor.get().submit(() -> {});
+        // 2. request create WispTask in the WispEngine A
+        submitDone.set(true);
+        // 3. telling WispEngine A's it's time to sleep
+
+        // 5. wait sleep done. If latch.countDown() is not invoked inner 10 seconds, TEST FAILURE.
+        assertTrue(latch.await(10, TimeUnit.SECONDS));
+    }
+}
diff --git a/test/jdk/com/alibaba/wisp/bug/SelectorInitCriticalTest.java b/test/jdk/com/alibaba/wisp/bug/SelectorInitCriticalTest.java
new file mode 100644
index 00000000000..fa7c82c2cb7
--- /dev/null
+++ b/test/jdk/com/alibaba/wisp/bug/SelectorInitCriticalTest.java
@@ -0,0 +1,32 @@
+/*
+ * @test
+ * @library /lib/testlibrary
+ * @summary Test the fix to NPE issue caused by unexpected co-routine yielding on synchronized(lock) in SelectorProvider.provider() during initialization of WispEngine
+ * @run main/othervm -XX:+EnableCoroutine -Dcom.alibaba.wisp.transparentWispSwitch=true -XX:+UseWispMonitor SelectorInitCriticalTest
+ * @run main/othervm -XX:+EnableCoroutine -Dcom.alibaba.wisp.transparentWispSwitch=true -XX:+UseWispMonitor -Dcom.alibaba.wisp.version=2 SelectorInitCriticalTest
+*/
+
+
+import java.lang.reflect.Field;
+import java.nio.channels.spi.SelectorProvider;
+import java.util.concurrent.CountDownLatch;
+
+public class SelectorInitCriticalTest {
+    public static void main(String[] args) throws Exception {
+        Field f = SelectorProvider.class.getDeclaredField("lock");
+        f.setAccessible(true);
+        Object selectorProviderLock = f.get(null);
+        CountDownLatch latch = new CountDownLatch(1);
+
+        Thread t = new Thread(latch::countDown);
+
+        synchronized (selectorProviderLock) {
+            t.start();
+            // Holding selectorProviderLock for a while which will eventually blocks the initialization of t' WispEngine
+            Thread.sleep(100);
+        }
+
+        latch.await();
+
+    }
+}
diff --git a/test/jdk/com/alibaba/wisp/bug/TestThreadStackTrace.sh b/test/jdk/com/alibaba/wisp/bug/TestThreadStackTrace.sh
new file mode 100644
index 00000000000..740cc98444d
--- /dev/null
+++ b/test/jdk/com/alibaba/wisp/bug/TestThreadStackTrace.sh
@@ -0,0 +1,134 @@
+#!/usr/bin/env bash
+#
+# @test
+# @summary test Thread.getStackTrace() in wisp transparentAsync model
+# @modules java.base/jdk.internal.misc
+# @modules java.base/com.alibaba.wisp.engine:+open
+# @run shell TestThreadStackTrace.sh
+#
+
+if [ "${TESTSRC}" = "" ]
+then
+    TESTSRC=${PWD}
+    echo "TESTSRC not set.  Using "${TESTSRC}" as default"
+fi
+echo "TESTSRC=${TESTSRC}"
+# determine platform dependant variables
+OS=`uname -s`
+case ${OS} in
+  Linux)
+    FS=/
+    ;;
+  *)
+    exit 1
+    ;;
+esac
+
+JAVA=${TESTJAVA}${FS}bin${FS}java
+JAVAC=${TESTJAVA}${FS}bin${FS}javac
+TEST_CLASS=TmpThreadStackTrace
+TEST_SOURCE=${TEST_CLASS}.java
+TEST_WISP_CONFIG=$(mktemp)
+
+###################################################################################
+
+cat > ${TEST_WISP_CONFIG} << EOF
+com.alibaba.wisp.biz.manage=TmpThreadStackTrace::foo\n
+EOF
+
+cat > ${TESTCLASSES}${FS}$TEST_SOURCE << EOF
+import com.alibaba.wisp.engine.WispEngine;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.atomic.AtomicBoolean;
+import jdk.internal.access.SharedSecrets;
+import java.util.concurrent.Executors;
+import java.util.concurrent.*;
+
+public class TmpThreadStackTrace {
+
+    public static Thread runningCoroutine;
+
+    public static void foo() throws Exception {
+        ExecutorService executor = Executors.newCachedThreadPool();
+        executor.execute(() -> {
+            try {
+                runningCoroutine = Thread.currentThread();
+                Thread.sleep(2_000L);
+            } catch (InterruptedException e) {
+                e.printStackTrace();
+            }
+        });
+        AtomicBoolean result = new AtomicBoolean(true);
+        CountDownLatch done = new CountDownLatch(1);
+        Thread mainThread = Thread.currentThread();
+        Thread[] ts = new Thread[1];
+        ts[0] = new Thread(() -> {
+            System.out.println("in managed!");
+            try {
+                if (ts[0].getStackTrace().length == 0 || mainThread.getStackTrace().length == 0
+                        || Thread.currentThread().getStackTrace().length == 0) {
+                    result.set(false);
+                }
+                runningCoroutine.getStackTrace();
+                // Should not reach here
+                result.set(false);
+            } catch (Exception e) {
+                if (!(e instanceof UnsupportedOperationException)) {
+                    result.set(false);
+                }
+            } finally {
+                done.countDown();
+            }
+        }, "test-thread");
+        ts[0].start();
+        done.await();
+        System.out.println("in main");
+        executor.shutdown();
+        if (!result.get()) {
+            throw new Error("test failure");
+        }
+    }
+
+    public static void main(String[] args) throws Exception {
+        final CountDownLatch latch = new CountDownLatch(1);
+        WispEngine.dispatch(() -> {
+            for (int i = 0; i < 5; i++) {
+                System.out.println(i);
+                Thread.currentThread().getStackTrace();
+            }
+            latch.countDown();
+        });
+        latch.await();
+        foo();
+        System.out.println("done");
+        System.exit(0);
+    }
+}
+EOF
+
+# Do compilation
+${JAVAC} --add-exports java.base/jdk.internal.misc=ALL-UNNAMED -cp ${TESTCLASSES} -d ${TESTCLASSES} ${TESTCLASSES}${FS}$TEST_SOURCE >> /dev/null 2>&1
+if [ $? != '0' ]
+then
+	printf "Failed to compile ${TESTCLASSES}${FS}${TEST_SOURCE}"
+	exit 1
+fi
+
+#run
+${JAVA} -XX:+PrintSafepointStatistics  -XX:PrintSafepointStatisticsCount=1 -Dcom.alibaba.wisp.config=${TEST_WISP_CONFIG} -XX:-UseBiasedLocking -XX:+EnableCoroutine -Dcom.alibaba.wisp.transparentWispSwitch=true  -Dcom.alibaba.wisp.enableThreadAsWisp=true -cp ${TESTCLASSES} ${TEST_CLASS} > output.txt  2>&1
+rm -f $TEST_WISP_CONFIG
+cat output.txt
+
+function assert()
+{
+    line=`cat output.txt | grep ThreadDump | wc -l`
+    echo $line
+    if [[ $line -eq "2" ]]; then
+        echo "success"
+    else
+        echo "failure"
+        exit -1
+    fi
+}
+
+assert
diff --git a/test/jdk/com/alibaba/wisp/bug/ThreadLockTest.java b/test/jdk/com/alibaba/wisp/bug/ThreadLockTest.java
new file mode 100644
index 00000000000..81fe5def1e2
--- /dev/null
+++ b/test/jdk/com/alibaba/wisp/bug/ThreadLockTest.java
@@ -0,0 +1,75 @@
+/*
+ * @test
+ * @summary Test fix of WispEngine block on Thread.class lock
+ * @modules java.base/java.lang:+open
+ * @run main/othervm -XX:-UseBiasedLocking -XX:+EnableCoroutine -XX:+UseWispMonitor -XX:+UnlockExperimentalVMOptions -XX:SyncKnobs="ReportSettings=1:QMode=1" -Dcom.alibaba.wisp.transparentWispSwitch=true ThreadLockTest
+ * @run main/othervm -XX:-UseBiasedLocking -XX:+EnableCoroutine -XX:+UseWispMonitor -XX:+UnlockExperimentalVMOptions -XX:SyncKnobs="ReportSettings=1:QMode=1" -Dcom.alibaba.wisp.transparentWispSwitch=true -Dcom.alibaba.wisp.version=2 ThreadLockTest
+ */
+
+import com.alibaba.wisp.engine.WispEngine;
+
+import java.lang.reflect.Method;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.atomic.AtomicReference;
+
+public class ThreadLockTest {
+    public static void main(String[] args) throws Exception {
+
+        CountDownLatch done = new CountDownLatch(1);
+        AtomicReference es = new AtomicReference<>(null);
+
+        synchronized (Thread.class) {
+            new Thread(() -> {
+                es.set(WispEngine.current());
+                try {
+                    Thread.sleep(1000);
+                } catch (InterruptedException e) {
+                    e.printStackTrace();
+                }
+            }, "ThreadIdGeneratorLockTest").start();
+
+            Thread.sleep(100); // make sure "ThreadIdGeneratorLockTest" already sleeping
+
+            /*
+                 if ((SyncFlags & 16) == 0 && nxt == NULL && _EntryList == NULL) {
+                    // Try to assume the role of responsible thread for the monitor.
+                    // CONSIDER:  ST vs CAS vs { if (Responsible==null) Responsible=Self }
+                     Atomic::cmpxchg_ptr (Self, &_Responsible, NULL) ;
+                 }
+                  ----------------
+                 if (_Responsible == Self || (SyncFlags & 1)) {
+                     use timeout park
+                 }
+                 add a waiter, let _Responsible != Self
+             */
+            new Thread(() -> {
+                try {
+                    Method m = Thread.class.getDeclaredMethod("nextThreadNum");
+                    m.setAccessible(true);
+                    m.invoke(null);
+                } catch (ReflectiveOperationException e) {
+                    throw new Error(e);
+                }
+            }, "waiter").start();
+            Thread.sleep(100); // make sure adding waiter is done
+
+            es.get().execute(() -> { // give an name "A"
+                Thread.currentThread(); // before fix, we'll hang here
+                done.countDown();
+            });
+
+            // objectMonitor::EnterI generally puts waiter to head, now _cxq is:
+            // "A" (os_park) --> "A" (wisp_park) --> "waiter"
+            // set QMode=1 to reverse the queue
+            // _Entry_list is:
+            // "waiter" --> "A" (wisp_park) --> "A" (os_park)
+            Thread.sleep(100); // still hold lock 100 ms, make sure the lsat wisp blocked
+        }
+        // release:
+        // 1. wisp unpark "waiter"
+        // 2. wisp unpark "A" (wisp_park) (just after waiter ends)
+        // now "ThreadIdGeneratorLockTest" Thread is on os_park, test failed!
+
+        done.await();
+    }
+}
diff --git a/test/jdk/com/alibaba/wisp/bug/ThreadPoolFastShutdownBugTest.java b/test/jdk/com/alibaba/wisp/bug/ThreadPoolFastShutdownBugTest.java
new file mode 100644
index 00000000000..31b9632a712
--- /dev/null
+++ b/test/jdk/com/alibaba/wisp/bug/ThreadPoolFastShutdownBugTest.java
@@ -0,0 +1,26 @@
+/*
+ * @test
+ * @summary test shutdown a thread pool which contains non-fully-started thread
+ * @run main/othervm -XX:+EnableCoroutine -Dcom.alibaba.wisp.transparentWispSwitch=true -Dcom.alibaba.wisp.version=1 ThreadPoolFastShutdownBugTest
+ * @run main/othervm -XX:+EnableCoroutine -Dcom.alibaba.wisp.transparentWispSwitch=true -Dcom.alibaba.wisp.version=2 ThreadPoolFastShutdownBugTest
+
+ */
+
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+
+public class ThreadPoolFastShutdownBugTest {
+    public static void main(String[] args) {
+        ExecutorService executorService = Executors.newSingleThreadExecutor();
+        executorService.submit(new InitThread());
+        executorService.shutdown();
+    }
+
+    static class InitThread implements Runnable {
+
+        @Override
+        public void run() {
+
+        }
+    }
+}
diff --git a/test/jdk/com/alibaba/wisp/bug/WispEngineCriticalSectionTest.java b/test/jdk/com/alibaba/wisp/bug/WispEngineCriticalSectionTest.java
new file mode 100644
index 00000000000..f96f46ebd04
--- /dev/null
+++ b/test/jdk/com/alibaba/wisp/bug/WispEngineCriticalSectionTest.java
@@ -0,0 +1,88 @@
+/*
+ * @test
+ * @summary Test case for fix a deadlock caused by critical section of WispEngine
+ * @modules java.base/jdk.internal.misc
+ * @modules java.base/com.alibaba.wisp.engine:+open
+ * @run main/othervm   -XX:-UseBiasedLocking -XX:+EnableCoroutine -XX:+UseWispMonitor -Dcom.alibaba.wisp.transparentWispSwitch=true -Dcom.alibaba.globalPoller=false WispEngineCriticalSectionTest
+ */
+
+
+import com.alibaba.wisp.engine.WispEngine;
+import jdk.internal.access.SharedSecrets;
+
+import java.lang.reflect.Field;
+import java.net.Socket;
+import java.nio.channels.spi.AbstractSelector;
+import java.util.HashSet;
+import java.util.concurrent.CountDownLatch;
+
+public class WispEngineCriticalSectionTest {
+    public static void main(String[] args) throws Exception {
+        // do blocking operation and register to engine
+        Socket so = new Socket("www.example.com", 80);
+
+        so.close(); // put into engine's cancel queue
+
+        if (!Class.forName("com.alibaba.wisp.engine.ScheduledWispEngine").isInstance(WispEngine.current())) {
+            return;
+        }
+        Field f = Class.forName("com.alibaba.wisp.engine.ScheduledWispEngine").getDeclaredField("selector");
+        f.setAccessible(true);
+        AbstractSelector selector = (AbstractSelector) f.get(WispEngine.current());
+        f = AbstractSelector.class.getDeclaredField("cancelledKeys");
+        f.setAccessible(true);
+        final HashSet cancelSet = (HashSet) f.get(selector);
+
+
+        // made an runnable wisp
+        CountDownLatch cd = new CountDownLatch(1);
+        WispEngine.dispatch(() -> { // coroutine A
+            try {
+                cd.await();
+            } catch (InterruptedException e) {
+                e.printStackTrace();
+            }
+
+            // finally let the engine select...
+            try {
+                Thread.sleep(20);
+            } catch (InterruptedException e) {
+                e.printStackTrace();
+            }
+            //
+        });
+        cd.countDown();
+        // coroutine A is runnable
+
+
+        CountDownLatch lock = new CountDownLatch(1);
+        new Thread(() -> {
+            synchronized (lock) {
+                lock.countDown();
+                try {
+                    Thread.sleep(30);
+                } catch (InterruptedException e) {
+                    e.printStackTrace();
+                }
+            }
+        }).start();
+
+        lock.await();
+        // now above thread holds lock(CountDownLatch).
+
+        SharedSecrets.getWispEngineAccess().runInCritical(() -> {
+            //deadlock if we don't put the below logic inside critical section of WispEngine
+            synchronized (cancelSet) {
+                synchronized (lock) {
+            /*
+                If without protection of critical section, attempting to acquire lock will yield to coroutine A,
+                which continues after "cd.await();", then deadlocks when the processDeregisterQueue
+                (trigger by Thread.sleep) is trying to acquire cancelSet lock
+            */
+
+                }
+            }
+            return null;
+        });
+    }
+}
diff --git a/test/jdk/com/alibaba/wisp/bug/WispSelectorReadyOpsTest.java b/test/jdk/com/alibaba/wisp/bug/WispSelectorReadyOpsTest.java
new file mode 100644
index 00000000000..657ba2a42c2
--- /dev/null
+++ b/test/jdk/com/alibaba/wisp/bug/WispSelectorReadyOpsTest.java
@@ -0,0 +1,45 @@
+/*
+ * @test
+ * @summary ensure nio program call SelectionKey.is{}able() and got correct result.
+ * @run main/othervm -XX:+EnableCoroutine -Dcom.alibaba.wisp.transparentWispSwitch=true WispSelectorReadyOpsTest
+ * @run main/othervm -XX:+EnableCoroutine -Dcom.alibaba.wisp.transparentWispSwitch=true -Dcom.alibaba.wisp.version=2 WispSelectorReadyOpsTest
+*/
+
+import java.net.InetSocketAddress;
+import java.net.Socket;
+import java.nio.channels.SelectionKey;
+import java.nio.channels.Selector;
+import java.nio.channels.ServerSocketChannel;
+import java.nio.channels.SocketChannel;
+
+public class WispSelectorReadyOpsTest {
+    public static void main(String[] args) throws Exception {
+        Selector selector = Selector.open();
+        ServerSocketChannel sch = ServerSocketChannel.open();
+        sch.configureBlocking(false);
+        sch.bind(new InetSocketAddress(54442));
+        sch.register(selector, sch.validOps());
+
+        Socket so = new Socket("127.0.0.1", 54442);
+
+        selector.selectNow();
+        SelectionKey sk = (SelectionKey) selector.selectedKeys().toArray()[0];
+        if (!sk.isAcceptable()) {
+            throw new Error("fail");
+        }
+
+        SocketChannel sc = sch.accept();
+        sc.configureBlocking(false);
+        sc.register(selector, sc.validOps());
+
+        so.getOutputStream().write("123".getBytes());
+
+        selector.selectedKeys().clear();
+        selector.selectNow();
+        sk = (SelectionKey) selector.selectedKeys().toArray()[0];
+        if (!sk.isReadable()) {
+            throw new Error("fail");
+        }
+    }
+
+}
diff --git a/test/jdk/com/alibaba/wisp/bug/WispSocketLeakWhenConnectTimeoutTest.java b/test/jdk/com/alibaba/wisp/bug/WispSocketLeakWhenConnectTimeoutTest.java
new file mode 100644
index 00000000000..725832626f8
--- /dev/null
+++ b/test/jdk/com/alibaba/wisp/bug/WispSocketLeakWhenConnectTimeoutTest.java
@@ -0,0 +1,28 @@
+/*
+ * @test
+ * @library /lib/testlibrary
+ * @summary test the fix to fd leakage when socket connect timeout
+ * @run main/othervm -XX:+EnableCoroutine -Dcom.alibaba.wisp.transparentWispSwitch=true -Dcom.alibaba.wisp.version=2 WispSocketLeakWhenConnectTimeoutTest
+*/
+
+import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.net.Socket;
+import java.net.SocketTimeoutException;
+
+import static jdk.testlibrary.Asserts.assertTrue;
+
+public class WispSocketLeakWhenConnectTimeoutTest {
+    public static void main(String[] args) throws IOException {
+        Socket so = new Socket();
+        boolean timeout = false;
+        try {
+            so.connect(new InetSocketAddress("www.facebook.com", 80), 5);
+        } catch (SocketTimeoutException e) {
+            assertTrue(so.isClosed());
+            timeout = true;
+        }
+
+        assertTrue(timeout, "SocketTimeoutException should been thrown");
+    }
+}
diff --git a/test/jdk/com/alibaba/wisp/close/WispDestroyTest.java b/test/jdk/com/alibaba/wisp/close/WispDestroyTest.java
new file mode 100644
index 00000000000..0ac1dc81a78
--- /dev/null
+++ b/test/jdk/com/alibaba/wisp/close/WispDestroyTest.java
@@ -0,0 +1,38 @@
+/*
+ * @test
+ * @summary Test WispEngine's destroy
+ * @modules java.base/com.alibaba.wisp.engine:+open
+ * @run main/othervm -XX:+EnableCoroutine -Dcom.alibaba.wisp.transparentWispSwitch=true WispDestroyTest
+ * @run main/othervm -XX:+EnableCoroutine -Dcom.alibaba.wisp.transparentWispSwitch=true -Dcom.alibaba.wisp.version=2 WispDestroyTest
+*/
+
+import com.alibaba.wisp.engine.WispEngine;
+
+import java.lang.reflect.Field;
+import java.nio.channels.Selector;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.nio.channels.ClosedSelectorException;
+
+public class WispDestroyTest {
+
+    static WispEngine e;
+
+    public static void main(String[] args) throws Exception {
+        CountDownLatch l = new CountDownLatch(1);
+
+        new Thread(() -> {
+            e = WispEngine.current();
+            l.countDown();
+        }).start();
+
+        l.await();
+
+        Thread.sleep(1000); // ensure Thread.exit() executed
+
+        Field f = WispEngine.class.getDeclaredField("terminated");
+        f.setAccessible(true);
+
+        if (!f.getBoolean(e)) throw new Error("resource leak!");
+    }
+}
diff --git a/test/jdk/com/alibaba/wisp/env/CtxClassLoaderIsolateTest.java b/test/jdk/com/alibaba/wisp/env/CtxClassLoaderIsolateTest.java
new file mode 100644
index 00000000000..9adb6d19e98
--- /dev/null
+++ b/test/jdk/com/alibaba/wisp/env/CtxClassLoaderIsolateTest.java
@@ -0,0 +1,26 @@
+/*
+ * @test
+ * @library /lib/testlibrary
+ * @summary Verify the context class loader isolation per co-routine
+ * @run main/othervm -XX:+EnableCoroutine -Dcom.alibaba.wisp.transparentWispSwitch=true CtxClassLoaderIsolateTest
+ * @run main/othervm -XX:+EnableCoroutine -Dcom.alibaba.wisp.transparentWispSwitch=true -Dcom.alibaba.wisp.version=2 CtxClassLoaderIsolateTest
+*/
+
+
+import com.alibaba.wisp.engine.WispEngine;
+
+import static jdk.testlibrary.Asserts.assertTrue;
+
+public class CtxClassLoaderIsolateTest {
+    public static void main(String[] args) throws Exception {
+        ClassLoader loader0 = Thread.currentThread().getContextClassLoader();
+        WispEngine.dispatch(() -> Thread.currentThread().setContextClassLoader(new ClassLoader() {
+            @Override
+            public Class loadClass(String name) throws ClassNotFoundException {
+                return super.loadClass(name);
+            }
+        }));
+
+        assertTrue(Thread.currentThread().getContextClassLoader() == loader0);
+    }
+}
diff --git a/test/jdk/com/alibaba/wisp/io/CreateFdOnDemandTest.java b/test/jdk/com/alibaba/wisp/io/CreateFdOnDemandTest.java
new file mode 100644
index 00000000000..e9731ac78f7
--- /dev/null
+++ b/test/jdk/com/alibaba/wisp/io/CreateFdOnDemandTest.java
@@ -0,0 +1,61 @@
+/*
+ * @test
+ * @library /lib/testlibrary
+ * @summary Test fix of unconnected Socket fd leak.
+ * @run main/othervm -XX:+EnableCoroutine -Dcom.alibaba.wisp.transparentWispSwitch=true CreateFdOnDemandTest
+ * @run main/othervm -XX:+EnableCoroutine -Dcom.alibaba.wisp.transparentWispSwitch=true -Dcom.alibaba.wisp.version=2 CreateFdOnDemandTest
+*/
+
+import java.io.File;
+import java.net.DatagramSocket;
+import java.net.ServerSocket;
+import java.net.Socket;
+
+import com.alibaba.wisp.engine.WispEngine;
+
+import static jdk.testlibrary.Asserts.assertEQ;
+import static jdk.testlibrary.Asserts.assertTrue;
+
+public class CreateFdOnDemandTest {
+    public static void main(String[] args) throws Exception {
+
+        assertEQ(countFd(), countFd());
+        Socket so = new Socket();
+        so.setReuseAddress(true);
+        so.close();
+        ServerSocket sso = new ServerSocket();
+        sso.setReuseAddress(true);
+        sso.close();
+        DatagramSocket ds = new DatagramSocket(null);
+        ds.setReuseAddress(true);
+        ds.close();
+
+
+        final int nfd0 = countFd();
+        so = new Socket();
+        assertEQ(countFd(), nfd0);
+        sso = new ServerSocket();
+        assertEQ(countFd(), nfd0);
+        ds = new DatagramSocket(null);
+        assertTrue(WispEngine.transparentWispSwitch() && countFd() == nfd0); // if -Dcom.alibaba.wisp.transparentWispSwitch=false, fail
+
+        so.setReuseAddress(true);
+        assertEQ(countFd(), nfd0 + 1);
+        sso.setReuseAddress(true);
+        assertEQ(countFd(), nfd0 + 2);
+        ds.setReuseAddress(true);
+        assertEQ(countFd(), nfd0 + 3);
+
+        so.close();
+        assertEQ(countFd(), nfd0 + 2);
+        sso.close();
+        assertEQ(countFd(), nfd0 + 1);
+        ds.close();
+        assertEQ(countFd(), nfd0);
+    }
+
+    private static int countFd() {
+        File f = new File("/proc/self/fd");
+        return f.list().length;
+    }
+}
diff --git a/test/jdk/com/alibaba/wisp/io/DatagramSocketTest.java b/test/jdk/com/alibaba/wisp/io/DatagramSocketTest.java
new file mode 100644
index 00000000000..e580720def0
--- /dev/null
+++ b/test/jdk/com/alibaba/wisp/io/DatagramSocketTest.java
@@ -0,0 +1,87 @@
+/*
+ * @test
+ * @library /lib/testlibrary
+ * @summary Test WispEngine's DatagramSocket, InitialDirContext use dup socket to query dns.
+ * @modules java.base/jdk.internal.misc
+ * @run main/othervm -XX:+EnableCoroutine -Dcom.alibaba.wisp.transparentWispSwitch=true DatagramSocketTest
+ * @run main/othervm -XX:+EnableCoroutine -Dcom.alibaba.wisp.transparentWispSwitch=true -Dcom.alibaba.wisp.version=2 DatagramSocketTest
+*/
+
+
+import com.alibaba.wisp.engine.WispEngine;
+import jdk.internal.access.SharedSecrets;
+
+import javax.naming.directory.InitialDirContext;
+import java.io.IOException;
+import java.net.*;
+import java.util.concurrent.CountDownLatch;
+import static jdk.testlibrary.Asserts.*;
+
+public class DatagramSocketTest {
+    public static void main(String[] args) throws Exception {
+        for (int i = 0; i < 10; i++) {
+            WispEngine.dispatch(DatagramSocketTest::foo);
+        }
+
+        SharedSecrets.getWispEngineAccess().eventLoop();
+        testSetAndGetReceiveBufferSize();
+        testSendAndReceive();
+    }
+
+    static public void foo() {
+        try {
+            InitialDirContext dirCtx = new InitialDirContext();
+            System.out.println(dirCtx.getAttributes("dns:/www.tmall.com"));
+        } catch (Throwable e) {
+            throw new Error("query dns error");
+        }
+    }
+
+    static public void testSetAndGetReceiveBufferSize() throws Exception {
+        InetAddress host = InetAddress.getByName("localhost");
+        int port = 9527;
+        DatagramSocket so = new DatagramSocket(port);
+        so.setSoTimeout(1_000);
+        System.out.println("Receive buffer size: " + so.getReceiveBufferSize());
+        so.setReceiveBufferSize(1024 * 32);
+        System.out.println("Receive buffer size: " + so.getReceiveBufferSize());
+        assertTrue(so.getReceiveBufferSize() == 1024 * 32);
+        so.close();
+    }
+
+    static public void testSendAndReceive() throws Exception {
+        CountDownLatch count = new CountDownLatch(1);
+        InetAddress host = InetAddress.getByName("localhost");
+        int port = 9527;
+        DatagramSocket so = new DatagramSocket(port);
+        so.setSoTimeout(1_000);
+        final int loop = 1024;
+        Thread receiver = new Thread(() -> {
+            int received = 0;
+            try {
+                byte[] buf = new byte[1024 * 32];
+                DatagramPacket dp = new DatagramPacket(buf, buf.length);
+                count.countDown();
+                for (int i = 0; i < loop; i++) {
+                    received++;
+                    so.receive(dp);
+                }
+            } catch (SocketTimeoutException e) {
+                e.printStackTrace();
+                System.out.println("received packets: " + received);
+                // We at least received 64 packets before timeout
+                assertTrue(received > 64);
+            } catch (IOException e) {
+                throw new Error(e);
+            }
+        });
+        receiver.start();
+        count.await();
+        for (int i = 0; i < loop; i++ ) {
+            byte[] buf = new byte[1024 * 32];
+            DatagramPacket dp = new DatagramPacket(buf, buf.length, host, port);
+            so.send(dp);
+        }
+        receiver.join();
+    }
+}
diff --git a/test/jdk/com/alibaba/wisp/io/GlobalPollerTest.java b/test/jdk/com/alibaba/wisp/io/GlobalPollerTest.java
new file mode 100644
index 00000000000..9189516f4f2
--- /dev/null
+++ b/test/jdk/com/alibaba/wisp/io/GlobalPollerTest.java
@@ -0,0 +1,53 @@
+/*
+ * @test
+ * @library /lib/testlibrary
+ * @summary Test for Global Poller
+ * @modules java.base/jdk.internal.misc
+ * @modules java.base/sun.nio.ch
+ * @modules java.base/com.alibaba.wisp.engine:+open
+ * @run main/othervm -XX:+EnableCoroutine -Dcom.alibaba.transparentAsync=true GlobalPollerTest
+*/
+
+import com.alibaba.wisp.engine.WispEngine;
+import com.alibaba.wisp.engine.WispPoller;
+import com.alibaba.wisp.engine.WispTask;
+import jdk.internal.access.SharedSecrets;
+import jdk.internal.access.WispEngineAccess;
+import sun.nio.ch.SelChImpl;
+
+import java.lang.reflect.Field;
+import java.net.Socket;
+import java.nio.channels.SelectionKey;
+import java.nio.channels.SocketChannel;
+
+import static jdk.testlibrary.Asserts.assertTrue;
+
+public class GlobalPollerTest {
+    private static WispEngineAccess access = SharedSecrets.getWispEngineAccess();
+
+    public static void main(String[] args) throws Exception {
+
+
+        Socket so = new Socket("www.example.com", 80);
+        so.getOutputStream().write("NOP\n\r\n\r".getBytes());
+        // now server returns the data..
+        // so is readable
+        // current task is interested in read event.
+        SocketChannel ch = so.getChannel();
+        access.registerEvent(ch, SelectionKey.OP_READ);
+
+        Field f = Class.forName("com.alibaba.wisp.engine.WispPoller").getDeclaredField("fd2TaskLow");
+        f.setAccessible(true);
+        WispTask[] fd2TaskLow = (WispTask[]) f.get(WispPoller.INSTANCE);
+        int fd = ((SelChImpl) ch).getFDVal();
+        assertTrue(fd2TaskLow[fd] != null);
+
+        System.out.println(fd2TaskLow[fd].getName());
+
+        access.park(-1);
+
+        assertTrue(fd2TaskLow[fd] == null);
+
+        so.close();
+    }
+}
diff --git a/test/jdk/com/alibaba/wisp/io/ReuseUdpSocektBufTest.java b/test/jdk/com/alibaba/wisp/io/ReuseUdpSocektBufTest.java
new file mode 100644
index 00000000000..44ceceecca4
--- /dev/null
+++ b/test/jdk/com/alibaba/wisp/io/ReuseUdpSocektBufTest.java
@@ -0,0 +1,72 @@
+/*
+ * @test
+ * @library /lib/testlibrary
+ * @summary test reuse WispUdpSocket buffer
+ * @run main/othervm -XX:+EnableCoroutine -Dcom.alibaba.wisp.transparentWispSwitch=true ReuseUdpSocektBufTest
+ */
+
+import java.net.DatagramPacket;
+import java.net.DatagramSocket;
+import java.net.InetAddress;
+
+import static jdk.testlibrary.Asserts.assertTrue;
+
+
+public class ReuseUdpSocektBufTest {
+
+	static String msgs[] = {"Hello World", "Java", "Good Bye"};
+	static int port;
+
+	static boolean success = true;
+
+	static class ServerThread extends Thread{
+		DatagramSocket ds;
+		public ServerThread() {
+			try {
+				ds = new DatagramSocket();
+				port = ds.getLocalPort();
+			} catch (Exception e) {
+				throw new RuntimeException(e.getMessage());
+			}
+		}
+
+		public void run() {
+			byte b[] = new byte[100];
+			DatagramPacket dp = new DatagramPacket(b,b.length);
+			while (true) {
+				try {
+					ds.receive(dp);
+					String reply = new String(dp.getData(), dp.getOffset(), dp.getLength());
+					ds.send(new DatagramPacket(reply.getBytes(),reply.length(),
+							dp.getAddress(),dp.getPort()));
+					if (reply.equals(msgs[msgs.length-1])) {
+						break;
+					}
+				} catch (Exception e) {
+					success = false;
+				}
+			}
+			ds.close();
+		}
+	}
+
+	public static void main(String args[]) throws Exception {
+		ServerThread st = new ServerThread();
+		st.start();
+		DatagramSocket ds = new DatagramSocket();
+		byte b[] = new byte[100];
+		DatagramPacket dp = new DatagramPacket(b,b.length);
+		for (int i = 0; i < msgs.length; i++) {
+			ds.send(new DatagramPacket(msgs[i].getBytes(),msgs[i].length(),
+					InetAddress.getLocalHost(),
+					port));
+			ds.receive(dp);
+			if (!msgs[i].equals(new String(dp.getData(), dp.getOffset(), dp.getLength()))) {
+				success = false;
+			}
+		}
+		ds.close();
+		assertTrue(success);
+		System.out.println("Test Passed!!!");
+	}
+}
diff --git a/test/jdk/com/alibaba/wisp/io/SelectorRebuildTest.java b/test/jdk/com/alibaba/wisp/io/SelectorRebuildTest.java
new file mode 100644
index 00000000000..435a9ac1d2b
--- /dev/null
+++ b/test/jdk/com/alibaba/wisp/io/SelectorRebuildTest.java
@@ -0,0 +1,83 @@
+import com.alibaba.wisp.engine.WispEngine;
+
+import java.io.*;
+import java.net.ServerSocket;
+import java.net.Socket;
+import java.util.concurrent.CountDownLatch;
+
+/* @test
+ * @summary Selector rebuild test
+ * @run main/othervm -XX:+EnableCoroutine -XX:+UseWispMonitor -Dcom.alibaba.transparentAsync=true -Dcom.alibaba.shiftThreadModel=true -Dcom.alibaba.globalPoller=false SelectorRebuildTest
+ */
+
+public class SelectorRebuildTest {
+
+    public static void main(String[] args) throws Exception {
+
+        startNetServer();
+        Thread wispThread;
+        CountDownLatch latch = new CountDownLatch(1);
+
+        wispThread = new Thread(() -> {
+            WispEngine.dispatch(new Runnable() {
+                @Override
+                public void run() {
+                    doNetIO();
+                    latch.countDown();
+                }
+            });
+        }, "SelectorThread");
+        wispThread.start();
+
+        Thread.sleep(2_000L);
+
+        for(int i = 0; i < 1000; i++) {
+            wispThread.interrupt();
+            Thread.sleep(1);
+        }
+
+        // wait task complete
+        latch.await();
+    }
+
+    private static ServerSocket ss;
+    private static final int PORT = 23000;
+    private static final int BUFFER_SIZE = 1024;
+
+    private static void startNetServer() throws IOException {
+        ss = new ServerSocket(PORT);
+        Thread t = new Thread(() -> {
+            try {
+                while (true) {
+                    Socket cs = ss.accept();
+                    OutputStream os = cs.getOutputStream();
+                    try {
+                        Thread.sleep(10_000L);
+                    } catch (InterruptedException e) {
+                    }
+                    os.write(new byte[BUFFER_SIZE]);
+                    os.flush();
+                    os.close();
+                    cs.close();
+                }
+            } catch (IOException e) {
+                e.printStackTrace();
+            }
+        });
+        t.setDaemon(true);
+        t.start();
+    }
+
+    private static void doNetIO() {
+        try {
+            Socket so = new Socket("localhost", PORT);
+            InputStream is = so.getInputStream();
+            int r = is.read(new byte[BUFFER_SIZE]);
+            is.close();
+            so.close();
+        } catch (IOException e) {
+            e.printStackTrace();
+            throw new Error(e);
+        }
+    }
+}
diff --git a/test/jdk/com/alibaba/wisp/io/SocketTest.java b/test/jdk/com/alibaba/wisp/io/SocketTest.java
new file mode 100644
index 00000000000..babda0d5287
--- /dev/null
+++ b/test/jdk/com/alibaba/wisp/io/SocketTest.java
@@ -0,0 +1,96 @@
+/*
+ * @test
+ * @summary Test WispEngine's Socket
+ * @run main/othervm -XX:+EnableCoroutine -Dcom.alibaba.wisp.transparentWispSwitch=true SocketTest
+ * @run main/othervm -XX:+EnableCoroutine -Dcom.alibaba.wisp.transparentWispSwitch=true -Dcom.alibaba.wisp.version=2 SocketTest
+*/
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.net.ServerSocket;
+import java.net.Socket;
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+import java.util.List;
+
+public class SocketTest {
+
+    public static void main(String[] args) throws IOException {
+        System.out.println(mkSocketPair()); // test accept() and connect()
+        testData();
+        testBlock();
+    }
+
+    static private void testBlock() throws IOException {
+        List sop = mkSocketPair();
+        new Thread(() -> {
+            try {
+                Socket so = sop.get(0);
+                Thread.sleep(100);
+                so.getOutputStream().write(new byte[2]);
+
+                so.close();
+            } catch (Exception e) {
+                throw new Error(e);
+            }
+        }).start();
+
+        Socket so = sop.get(1);
+
+        long now = System.currentTimeMillis();
+        if (2 != so.getInputStream().read(new byte[10])) {
+            throw new Error("read error");
+        }
+        if (Math.abs(System.currentTimeMillis() - now - 100) > 5)
+            throw new Error("not wake as expected");
+    }
+
+    static private void testData() throws IOException {
+        List sop = mkSocketPair();
+
+        new Thread(() -> {
+            try {
+                Socket so = sop.get(0);
+                OutputStream os = so.getOutputStream();
+                byte buf[] = new byte[4];
+                ByteBuffer bb = ByteBuffer.wrap(buf);
+                for (int i = 0; i < 10; i++) {
+                    bb.clear();
+                    bb.putInt(i);
+                    os.write(buf);
+                }
+                so.close();
+            } catch (Exception e) {
+                throw new Error(e);
+            }
+        }).start();
+        Socket so = sop.get(1);
+        InputStream is = so.getInputStream();
+        byte buf[] = new byte[4];
+        ByteBuffer bb = ByteBuffer.wrap(buf);
+        for (int i = 0; true; i++) {
+            bb.clear();
+            if (4 != is.read(buf)) {
+                if (i == 10) {
+                    so.close();
+                    break; // ok here
+                } else {
+                    throw new Error("read error");
+                }
+            }
+            if (bb.getInt() != i)
+                throw new Error("data error");
+        }
+
+    }
+
+    static private List mkSocketPair() throws IOException {
+        ServerSocket ss = new ServerSocket(13000);
+        Socket so = new Socket("localhost", 13000);
+        Socket so1 = ss.accept();
+        ss.close();
+
+        return Arrays.asList(so, so1);
+    }
+}
diff --git a/test/jdk/com/alibaba/wisp/lock/AQSTest.java b/test/jdk/com/alibaba/wisp/lock/AQSTest.java
new file mode 100644
index 00000000000..cdba42525e8
--- /dev/null
+++ b/test/jdk/com/alibaba/wisp/lock/AQSTest.java
@@ -0,0 +1,51 @@
+/*
+ * @test
+ * @summary Test AQS: CountDownLatch is implement by AQS
+ * @modules java.base/jdk.internal.misc
+ * @run main/othervm -XX:+EnableCoroutine -Dcom.alibaba.wisp.transparentWispSwitch=true AQSTest
+ * @run main/othervm -XX:+EnableCoroutine -Dcom.alibaba.wisp.transparentWispSwitch=true -Dcom.alibaba.wisp.version=2 AQSTest
+*/
+
+
+import com.alibaba.wisp.engine.WispEngine;
+import com.alibaba.wisp.engine.WispTask;
+import jdk.internal.access.SharedSecrets;
+
+import java.util.concurrent.CountDownLatch;
+
+public class AQSTest {
+    static CountDownLatch cd = new CountDownLatch(1);
+    static CountDownLatch cd2 = new CountDownLatch(1);
+    public static void main(String[] args) {
+        WispEngine.dispatch(() -> {
+            long start = System.currentTimeMillis();
+            try {
+                cd.await();
+                assertInterval(start, 100, 5);
+            } catch (InterruptedException e) {
+                throw new Error();
+            }
+        });
+        WispEngine.dispatch(() -> {
+            long start = System.currentTimeMillis();
+            try {
+                cd2.await();
+                assertInterval(start, 200, 5);
+            } catch (InterruptedException e) {
+                throw new Error();
+            }
+        });
+
+
+        SharedSecrets.getWispEngineAccess().sleep(100);
+        cd.countDown();
+        SharedSecrets.getWispEngineAccess().sleep(100);
+        cd2.countDown();
+        SharedSecrets.getWispEngineAccess().sleep(5);
+    }
+
+    public static void assertInterval(long start, int diff, int bias) {
+        if (Math.abs(System.currentTimeMillis() - start - diff) > bias)
+            throw new Error("not wakeup expected");
+    }
+}
diff --git a/test/jdk/com/alibaba/wisp/lock/ElisionSpinTest.java b/test/jdk/com/alibaba/wisp/lock/ElisionSpinTest.java
new file mode 100644
index 00000000000..4428d7a198a
--- /dev/null
+++ b/test/jdk/com/alibaba/wisp/lock/ElisionSpinTest.java
@@ -0,0 +1,42 @@
+/*
+ * @test
+ * @summary Test elision spin
+ * @modules java.base/jdk.internal.misc
+ * @library /lib/testlibrary
+ * @run main/othervm  -XX:+EnableCoroutine -Dcom.alibaba.wisp.transparentWispSwitch=true -Dcom.alibaba.wisp.useStealLock=false ElisionSpinTest
+ */
+
+import com.alibaba.wisp.engine.WispEngine;
+import jdk.internal.access.SharedSecrets;
+
+import java.util.concurrent.locks.Condition;
+import java.util.concurrent.locks.ReentrantLock;
+
+import static jdk.testlibrary.Asserts.assertFalse;
+import static jdk.testlibrary.Asserts.assertTrue;
+
+public class ElisionSpinTest {
+    public static void main(String[] args) {
+        assertFalse(SharedSecrets.getWispEngineAccess().hasMoreTasks());
+
+        ReentrantLock lock = new ReentrantLock();
+        Condition cond = lock.newCondition();
+
+        WispEngine.dispatch(() -> {
+            lock.lock();
+            try {
+                cond.awaitUninterruptibly();
+            } finally {
+                lock.unlock();
+            }
+        });
+
+        lock.lock();
+        try {
+            cond.signal();
+        } finally {
+            lock.unlock();
+        }
+        assertTrue(SharedSecrets.getWispEngineAccess().hasMoreTasks());
+    }
+}
diff --git a/test/jdk/com/alibaba/wisp/lock/LockTest.java b/test/jdk/com/alibaba/wisp/lock/LockTest.java
new file mode 100644
index 00000000000..18dd446cf3c
--- /dev/null
+++ b/test/jdk/com/alibaba/wisp/lock/LockTest.java
@@ -0,0 +1,72 @@
+/*
+ * @test
+ * @summary Test ReentrantLock in coroutine environment
+ * @modules java.base/jdk.internal.misc
+ * @run main/othervm  -XX:+EnableCoroutine  -Dcom.alibaba.wisp.transparentWispSwitch=true LockTest
+ * @run main/othervm  -XX:+EnableCoroutine  -Dcom.alibaba.wisp.transparentWispSwitch=true -Dcom.alibaba.wisp.version=2 LockTest
+*/
+
+
+import com.alibaba.wisp.engine.WispEngine;
+import com.alibaba.wisp.engine.WispTask;
+import jdk.internal.access.SharedSecrets;
+
+import java.util.concurrent.locks.Condition;
+import java.util.concurrent.locks.Lock;
+import java.util.concurrent.locks.ReentrantLock;
+
+public class LockTest {
+    static Lock lock = new ReentrantLock();
+    static Condition cond = lock.newCondition();
+
+    public static void main(String[] args) {
+        WispEngine.dispatch(LockTest::p1);
+        WispEngine.dispatch(LockTest::p2);
+
+        SharedSecrets.getWispEngineAccess().eventLoop();
+
+        System.out.println(lock);
+
+        new Thread(LockTest::p1).start();
+        SharedSecrets.getWispEngineAccess().sleep(1);
+        new Thread(LockTest::p2).start();
+    }
+
+    public static void assertInterval(long start, int diff, int bias) {
+        if (Math.abs(System.currentTimeMillis() - start - diff) > bias)
+            throw new Error("not wakeup expected");
+    }
+
+    private static void p1() {
+
+        lock.lock();
+        SharedSecrets.getWispEngineAccess().sleep(100);
+        try {
+            long start = System.currentTimeMillis();
+            cond.await();
+            assertInterval(start, 100, 5);
+
+        } catch (InterruptedException e) {
+            throw new Error();
+        } finally {
+            lock.unlock();
+        }
+
+
+    }
+
+    private static void p2() {
+
+        long start = System.currentTimeMillis();
+        lock.lock();
+        try {
+            assertInterval(start, 100, 5);
+            SharedSecrets.getWispEngineAccess().sleep(100);
+            cond.signal();
+        } finally {
+            lock.unlock();
+        }
+    }
+
+
+}
diff --git a/test/jdk/com/alibaba/wisp/lock/LockUninterruptiblyTest.java b/test/jdk/com/alibaba/wisp/lock/LockUninterruptiblyTest.java
new file mode 100644
index 00000000000..4cc9e34ca70
--- /dev/null
+++ b/test/jdk/com/alibaba/wisp/lock/LockUninterruptiblyTest.java
@@ -0,0 +1,57 @@
+/*
+ * @test
+ * @library /lib/testlibrary
+ * @summary Test to verify we are not spinning when we're trying to acquire monitor with interrupted status
+ * @modules java.base/com.alibaba.wisp.engine:+open
+ * @run main/othervm -XX:+EnableCoroutine -XX:+UseWispMonitor -Dcom.alibaba.wisp.transparentWispSwitch=true LockUninterruptiblyTest
+ * @run main/othervm -XX:+EnableCoroutine -XX:+UseWispMonitor -Dcom.alibaba.wisp.transparentWispSwitch=true -Dcom.alibaba.wisp.version=2 LockUninterruptiblyTest
+*/
+
+import com.alibaba.wisp.engine.WispEngine;
+
+import java.lang.reflect.Field;
+import java.util.concurrent.atomic.AtomicReference;
+
+import static jdk.testlibrary.Asserts.assertTrue;
+
+public class LockUninterruptiblyTest {
+
+    public static void main(String[] args) throws Exception {
+        Object lock = new Object();
+
+        synchronized (lock) {
+            AtomicReference engine = new AtomicReference<>();
+
+            new Thread(() -> {
+                engine.set(WispEngine.current());
+                Thread.currentThread().interrupt();
+                synchronized (lock) {
+                  // now we're consume 100% CPU
+                  // because nio can not block the
+                  // thread(coroutine) in a interrupted status,
+                  // as a result we're spinning in a fetching loop
+                }
+            }).start();
+
+            while (engine.get() == null) {
+            }
+
+            int loops = getEventLoops(engine.get());
+
+            Thread.sleep(100);
+
+            assertTrue(getEventLoops(engine.get()) < loops + 10);
+        }
+    }
+
+
+    private static int getEventLoops(WispEngine engine) throws Exception {
+        Field f = WispEngine.class.getDeclaredField("statistics");
+        f.setAccessible(true);
+        Object statistics = f.get(engine);
+        f = Class.forName(WispEngine.class.getName() + "$Statistics")
+                .getDeclaredField("eventLoops");
+        f.setAccessible(true);
+        return f.getInt(statistics);
+    }
+}
diff --git a/test/jdk/com/alibaba/wisp/lock/ParkNanoTest.java b/test/jdk/com/alibaba/wisp/lock/ParkNanoTest.java
new file mode 100644
index 00000000000..b24f33e491d
--- /dev/null
+++ b/test/jdk/com/alibaba/wisp/lock/ParkNanoTest.java
@@ -0,0 +1,27 @@
+/*
+ * @test
+ * @summary Test park nanos
+ * @run main/othervm -XX:+EnableCoroutine -Dcom.alibaba.wisp.transparentWispSwitch=true ParkNanoTest
+ * @run main/othervm -XX:+EnableCoroutine -Dcom.alibaba.wisp.transparentWispSwitch=true -Dcom.alibaba.wisp.version=2 ParkNanoTest
+*/
+
+
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+
+public class ParkNanoTest {
+    public static void main(String[] args) throws Exception {
+        doTest(400_000, 300_000);
+        doTest(800_000, 500_000);
+    }
+
+    private static void doTest(long wait, long expected) throws Exception {
+        CountDownLatch l = new CountDownLatch(1);
+        long start = System.nanoTime();
+        l.await(wait, TimeUnit.NANOSECONDS);
+        long diff = System.nanoTime() - start;
+
+        if (diff < expected)
+            throw new Error("wake up too early!");
+    }
+}
diff --git a/test/jdk/com/alibaba/wisp/lock/UnsafeParkTest.java b/test/jdk/com/alibaba/wisp/lock/UnsafeParkTest.java
new file mode 100644
index 00000000000..2edd745b01f
--- /dev/null
+++ b/test/jdk/com/alibaba/wisp/lock/UnsafeParkTest.java
@@ -0,0 +1,44 @@
+/*
+ * @test
+ * @library /lib/testlibrary
+ * @summary Test to verify we can do proper wisp scheduling while calling on Unsafe.park()
+ * @run main/othervm -XX:+EnableCoroutine -Dcom.alibaba.wisp.transparentWispSwitch=true UnsafeParkTest
+ * @run main/othervm -XX:+EnableCoroutine -Dcom.alibaba.wisp.transparentWispSwitch=true -Dcom.alibaba.wisp.version=2 UnsafeParkTest
+*/
+
+import com.alibaba.wisp.engine.WispEngine;
+import sun.misc.Unsafe;
+
+import java.lang.reflect.Field;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLong;
+
+import static jdk.testlibrary.Asserts.assertTrue;
+
+
+public class UnsafeParkTest {
+    public static void main(String[] args) throws Exception {
+        Field f = Unsafe.class.getDeclaredField("theUnsafe");
+        f.setAccessible(true);
+        Unsafe unsafe = (Unsafe) f.get(null);
+
+        AtomicLong awake = new AtomicLong();
+
+        WispEngine.dispatch(() -> {
+            try {
+                Thread.sleep(300);
+            } catch (InterruptedException e) {
+                e.printStackTrace();
+            }
+            awake.set(System.currentTimeMillis());
+        });
+
+        long start = System.currentTimeMillis();
+
+        unsafe.park(false, 1);
+        unsafe.park(false, TimeUnit.MILLISECONDS.toNanos(500));
+
+        assertTrue(Math.abs(awake.get() - start - 300) < 100,
+                "awake should be set before unsafe.park expired " + awake.get() + " " + start);
+    }
+}
diff --git a/test/jdk/com/alibaba/wisp/monitor/C2SyncMethodTest.java b/test/jdk/com/alibaba/wisp/monitor/C2SyncMethodTest.java
new file mode 100644
index 00000000000..7b6c9ec8756
--- /dev/null
+++ b/test/jdk/com/alibaba/wisp/monitor/C2SyncMethodTest.java
@@ -0,0 +1,19 @@
+/*
+ * @test
+ * @summary test to run a compiled/synchronized method with wisp enabled.
+ * @run main/othervm -XX:-UseBiasedLocking -XX:+EnableCoroutine -XX:+UseWispMonitor -Dcom.alibaba.wisp.transparentWispSwitch=true C2SyncMethodTest
+ * @run main/othervm -XX:-UseBiasedLocking -XX:+EnableCoroutine -XX:+UseWispMonitor -Dcom.alibaba.wisp.transparentWispSwitch=true -Dcom.alibaba.wisp.version=2 C2SyncMethodTest
+*/
+public class C2SyncMethodTest {
+    public synchronized static void main(String[] args) {
+        for (int i = 0; i < 1000000; i++) {
+            foo();
+        }
+    }
+
+    static volatile int n = 0;
+
+    synchronized static void foo() {
+        n++;
+    }
+}
diff --git a/test/jdk/com/alibaba/wisp/monitor/FinalizerTest.java b/test/jdk/com/alibaba/wisp/monitor/FinalizerTest.java
new file mode 100644
index 00000000000..0875993483c
--- /dev/null
+++ b/test/jdk/com/alibaba/wisp/monitor/FinalizerTest.java
@@ -0,0 +1,35 @@
+/*
+ * @test
+ * @summary Test unpark in a finalizer  thread.
+ * @run main/othervm   -XX:-UseBiasedLocking -XX:+EnableCoroutine -XX:+UseWispMonitor -Dcom.alibaba.wisp.transparentWispSwitch=true FinalizerTest
+ * @run main/othervm   -XX:-UseBiasedLocking -XX:+EnableCoroutine -XX:+UseWispMonitor -Dcom.alibaba.wisp.transparentWispSwitch=true -Dcom.alibaba.wisp.version=2 FinalizerTest
+*/
+
+public class FinalizerTest {
+    static final Foo lock = new Foo();
+    public static void main(String[] args) throws Exception {
+        new Foo();
+        new Thread(() -> {
+            try {
+                Thread.sleep(10);
+            } catch (InterruptedException e) {
+                e.printStackTrace();
+            }
+            System.gc();
+            System.runFinalization();
+        }, "call-System.gc()").start();
+        synchronized (lock) {
+            lock.wait();
+        }
+        System.out.println(Thread.currentThread().getName() + ": wait done");
+    }
+
+    static class Foo {
+        @Override
+        protected void finalize() throws Throwable {
+            synchronized (lock) {
+                lock.notify();
+            }
+        }
+    }
+}
diff --git a/test/jdk/com/alibaba/wisp/monitor/JNICriticalTest.java b/test/jdk/com/alibaba/wisp/monitor/JNICriticalTest.java
new file mode 100644
index 00000000000..394bd9909e5
--- /dev/null
+++ b/test/jdk/com/alibaba/wisp/monitor/JNICriticalTest.java
@@ -0,0 +1,58 @@
+/*
+ * @test
+ * @summary Test unpark in JNI critical case
+ * @modules java.base/jdk.internal.misc
+ * @run main/othervm   -XX:-UseBiasedLocking -XX:+EnableCoroutine -XX:+UseWispMonitor -Dcom.alibaba.wisp.transparentWispSwitch=true -Dcom.alibaba.wisp.version=2 JNICriticalTest
+ */
+
+
+import com.alibaba.wisp.engine.WispEngine;
+import jdk.internal.access.SharedSecrets;
+
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.stream.IntStream;
+import java.util.zip.Deflater;
+
+public class JNICriticalTest {
+    public static void main(String[] args) throws Exception {
+        CountDownLatch latch = new CountDownLatch(16);
+        AtomicInteger id = new AtomicInteger();
+        ExecutorService es = Executors.newCachedThreadPool(r -> {
+            Thread t = new Thread(r);
+            t.setName("JNICriticalTest" + id.getAndIncrement());
+            return t;
+        });
+        IntStream.range(0, 4).forEach(ign -> es.execute(() -> {
+            Deflater d = new Deflater();
+            AtomicInteger n = new AtomicInteger();
+            for (int i = 0; i < 4; i++) {
+                WispEngine.dispatch(() -> {
+                    while (n.get() < 1_000_000) {
+                        jniCritical(d);
+                        if (n.incrementAndGet() % 100000 == 0) {
+                            System.out.println(SharedSecrets.getJavaLangAccess().currentThread0().getName() + "/" + n.get() / 1000 +"k");
+                        }
+                    }
+                    latch.countDown();
+                });
+            }
+        }));
+        latch.await();
+    }
+
+
+    private static void jniCritical(Deflater d) {
+        d.reset();
+        d.setInput(bs);
+        d.finish();
+        byte[] out = new byte[4096 * 4];
+
+        d.deflate(out); // Enter the JNI critical block here.
+    }
+
+    static byte[] bs = new byte[12];
+}
diff --git a/test/jdk/com/alibaba/wisp/monitor/LazyUnparkBugTest.java b/test/jdk/com/alibaba/wisp/monitor/LazyUnparkBugTest.java
new file mode 100644
index 00000000000..59489f242d2
--- /dev/null
+++ b/test/jdk/com/alibaba/wisp/monitor/LazyUnparkBugTest.java
@@ -0,0 +1,55 @@
+/*
+ * @test
+ * @summary T12212948
+ * @library /lib/testlibrary
+ * @run main/othervm -XX:+EnableCoroutine -XX:+UseWispMonitor -Dcom.alibaba.wisp.transparentWispSwitch=true LazyUnparkBugTest
+ * @run main/othervm -XX:+EnableCoroutine -XX:+UseWispMonitor -Dcom.alibaba.wisp.transparentWispSwitch=true -Dcom.alibaba.wisp.version=2 LazyUnparkBugTest
+*/
+ 
+public class LazyUnparkBugTest {
+    private static volatile Object thunk = null;
+ 
+    public static void test() throws Exception {
+        thunk = null;
+        Thread t1 = new Thread(() -> {
+            try {
+                synchronized (LazyUnparkBugTest.class) {
+                    Thread.sleep(1_000L);
+                }
+                Object o;
+                do {
+                    o = thunk;
+                } while (o == null);
+            } catch (Exception e) {
+                e.printStackTrace();
+            }
+        });
+ 
+        Thread t2 = new Thread(() -> {
+            try {
+                Thread.sleep(5_0L);
+                synchronized (LazyUnparkBugTest.class) {
+                    System.out.println("in t2");
+                }
+            } catch (Exception e) {
+                e.printStackTrace();
+            }
+            thunk = new Object();
+        });
+        t1.start();
+        t2.start();
+        t1.join();
+        t2.join();
+    }
+ 
+    public static void main(String[] args) throws Exception {
+        long begin = System.currentTimeMillis();
+        test();
+        long end = System.currentTimeMillis();
+        System.out.println("cost : " + (end - begin) + " ms");
+        if ((end - begin) > 2000) {
+            throw new Error("this is bug " + (end - begin));
+        }
+    }
+ 
+}
diff --git a/test/jdk/com/alibaba/wisp/monitor/MultiThreadTest.java b/test/jdk/com/alibaba/wisp/monitor/MultiThreadTest.java
new file mode 100644
index 00000000000..0621411bcd6
--- /dev/null
+++ b/test/jdk/com/alibaba/wisp/monitor/MultiThreadTest.java
@@ -0,0 +1,146 @@
+/*
+ * @test
+ * @library /lib/testlibrary
+ * @summary Test object lock with coroutine
+ * @modules java.base/jdk.internal.misc
+ * @run main/othervm  -XX:-UseBiasedLocking -XX:+EnableCoroutine -XX:+UseWispMonitor -Dcom.alibaba.wisp.transparentWispSwitch=true -Dcom.alibaba.wisp.version=2 MultiThreadTest
+*/
+
+
+import com.alibaba.wisp.engine.WispEngine;
+import jdk.internal.access.SharedSecrets;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.locks.Condition;
+import java.util.concurrent.locks.Lock;
+import java.util.concurrent.locks.ReentrantLock;
+
+public class MultiThreadTest {
+    final static Runner[] runners = new Runner[8];
+    static boolean JUC;
+    static final int N = 40000;
+
+    public static void main(String[] args) throws Exception {
+        JUC = true;
+        System.out.println("JUC");
+        doTest();
+        JUC = false;
+        System.out.println("ObjectMonitor");
+        doTest();
+    }
+
+    private static void doTest() {
+        for (int i = 0; i < runners.length; i++) {
+            runners[i] = new Runner(i);
+        }
+        List plans = new ArrayList<>(Arrays.asList(
+                () -> { // only coroutine
+                    for (Runner runner1 : runners) {
+                        WispEngine.dispatch(runner1);
+                    }
+                },
+                () -> { // only thread
+                    int n = 0;
+                    for (Runner runner : runners) {
+                        new Thread(runner, "MP-THREAD-RUNNER-" + n++).start();
+                    }
+                },
+                () -> { //mixed
+                    int n = 0;
+                    for (int i = 0; i < runners.length; i += 2) {
+                        final int ci = i;
+                        new Thread(() -> {
+                            for (int j = ci; j < ci + 2 && j < runners.length; j++) {
+                                WispEngine.dispatch(runners[j]);
+                            }
+                        }, "MP-MIX-RUNNER-" + n++).start();
+                    }
+                }));
+        Collections.shuffle(plans);
+
+        plans.forEach(plan -> {
+            finishLatch = new CountDownLatch(runners.length);
+            current = 0;
+            long start = System.currentTimeMillis();
+            plan.run(); // create runners
+            try {
+                finishLatch.await();
+            } catch (InterruptedException e) {
+                e.printStackTrace();
+            }
+            System.out.println("cost = " + (System.currentTimeMillis() - start) + "ms");
+        });
+
+        System.out.println("-----------");
+    }
+
+
+    static volatile int current = 0;
+    static final Lock lock = new ReentrantLock();
+    static final Condition cond = lock.newCondition();
+    static CountDownLatch finishLatch;
+
+    static class Runner implements Runnable {
+
+        private final int ord;
+
+        public Runner(int ord) {
+            this.ord = ord;
+        }
+
+        @Override
+        public void run() {
+            while (current < N) {
+                if (JUC) {
+                    lock.lock();
+                    try {
+                        while (current % runners.length != ord) {
+                            try {
+                                //System.out.println("wait" + SharedSecrets.getJavaLangAccess().currentThread0().getName());
+                                cond.await();
+                            } catch (InterruptedException e) {
+                                e.printStackTrace();
+                            }
+                        }
+                        if (++current % 10000 == 0) // pass the token
+                            System.out.println(SharedSecrets.getJavaLangAccess().currentThread0().getName() + "\t" + current);
+                    } finally {
+                        lock.unlock();
+                    }
+                } else {
+                    synchronized (lock) {
+                        while (current % runners.length != ord) {
+                            try {
+                                lock.wait();
+                            } catch (InterruptedException e) {
+                                e.printStackTrace();
+                            }
+                        }
+                        if (++current % 10000 == 0) // pass the token
+                            System.out.println(SharedSecrets.getJavaLangAccess().currentThread0().getName() + "\t" + current);
+                    }
+                }
+
+                if (JUC) {
+                    lock.lock();
+                    try {
+                        cond.signalAll();
+                    } finally {
+                        lock.unlock();
+                    }
+                } else {
+                    synchronized (lock) {
+                        lock.notifyAll();
+                    }
+                }
+            }
+            System.out.println(SharedSecrets.getJavaLangAccess().currentThread0().getName() + " down"); 
+            finishLatch.countDown();
+        }
+    }
+}
+
diff --git a/test/jdk/com/alibaba/wisp/monitor/SynchronizedTest.java b/test/jdk/com/alibaba/wisp/monitor/SynchronizedTest.java
new file mode 100644
index 00000000000..f34a4faaa80
--- /dev/null
+++ b/test/jdk/com/alibaba/wisp/monitor/SynchronizedTest.java
@@ -0,0 +1,28 @@
+/*
+ * @test
+ * @summary Basic test for java primitive lock(synchronized)
+ * @run main/othervm -XX:-UseBiasedLocking -XX:+EnableCoroutine -XX:+UseWispMonitor -Dcom.alibaba.wisp.transparentWispSwitch=true SynchronizedTest
+ * @run main/othervm -XX:-UseBiasedLocking -XX:+EnableCoroutine -XX:+UseWispMonitor -Dcom.alibaba.wisp.transparentWispSwitch=true -Dcom.alibaba.wisp.version=2 SynchronizedTest
+*/
+
+import com.alibaba.wisp.engine.WispEngine;
+
+public class SynchronizedTest {
+    public static void main(String[] args) {
+
+        SynchronizedTest s = new SynchronizedTest();
+        WispEngine.dispatch(s::foo);
+        WispEngine.dispatch(s::bar);
+    }
+
+    private synchronized void foo() {
+        try {
+            Thread.sleep(10);
+        } catch (InterruptedException e) {
+            e.printStackTrace();
+        }
+    }
+
+    private synchronized void bar() {
+    }
+}
diff --git a/test/jdk/com/alibaba/wisp/monitor/WaitNotifyTest.java b/test/jdk/com/alibaba/wisp/monitor/WaitNotifyTest.java
new file mode 100644
index 00000000000..dbbb7c3088d
--- /dev/null
+++ b/test/jdk/com/alibaba/wisp/monitor/WaitNotifyTest.java
@@ -0,0 +1,65 @@
+/*
+ * @test
+ * @library /lib/testlibrary
+ * @summary Test Object.wait/notify with coroutine
+ * @run main/othervm  -XX:-UseBiasedLocking -XX:+EnableCoroutine -XX:+UseWispMonitor -Dcom.alibaba.wisp.transparentWispSwitch=true WaitNotifyTest
+ */
+
+import com.alibaba.wisp.engine.WispEngine;
+
+import java.util.concurrent.CountDownLatch;
+
+import static jdk.testlibrary.Asserts.assertEQ;
+
+public class WaitNotifyTest {
+    public static void main(String[] args) throws Exception {
+        for (int i = 0; i < 1; i++) {
+            WaitNotifyTest s = new WaitNotifyTest();
+            assertEQ(s.count++, 0);
+            WispEngine.dispatch(s::foo);
+            WispEngine.dispatch(s::bar);
+            assertEQ(s.count++, 3);
+            s.latch.countDown();
+            synchronized (s) {
+                while (s.finishCnt < 2) {
+                    s.wait();
+                }
+            }
+        }
+    }
+
+    private int count = 0;
+    private int finishCnt = 0;
+    private CountDownLatch latch = new CountDownLatch(1);
+    private boolean fooCond = false;
+
+    private synchronized void foo() {
+        assertEQ(count++, 1);
+        try {
+            latch.await();
+        } catch (InterruptedException e) {
+            e.printStackTrace();
+        }
+        assertEQ(count++, 4);
+        while (!fooCond) {
+            try {
+                wait();
+            } catch (InterruptedException e) {
+                e.printStackTrace();
+            }
+        }
+        assertEQ(count++, 6);
+        finishCnt++;
+        notifyAll();
+    }
+
+    private void bar() {
+        assertEQ(count++, 2);
+        synchronized (this) {
+            assertEQ(count++, 5);
+            finishCnt++;
+            fooCond = true;
+            notifyAll();
+        }
+    }
+}
diff --git a/test/jdk/com/alibaba/wisp/monitor/WispExitTest.java b/test/jdk/com/alibaba/wisp/monitor/WispExitTest.java
new file mode 100644
index 00000000000..2c6ea9d1cd3
--- /dev/null
+++ b/test/jdk/com/alibaba/wisp/monitor/WispExitTest.java
@@ -0,0 +1,11 @@
+/*
+ * @test
+ * @summary Ensure we can exit vm when -XX:+UseWispMonitor
+ * @run main/othervm -XX:-UseBiasedLocking -XX:+EnableCoroutine -XX:+UseWispMonitor -Dcom.alibaba.wisp.transparentWispSwitch=true WispExitTest
+ * @run main/othervm -XX:-UseBiasedLocking -XX:+EnableCoroutine -XX:+UseWispMonitor -Dcom.alibaba.wisp.transparentWispSwitch=true -Dcom.alibaba.wisp.version=2 WispExitTest
+*/
+public class WispExitTest {
+    public static void main(String[] args) throws Exception {
+
+    }
+}
diff --git a/test/jdk/com/alibaba/wisp/thread/DisableThreadAsWispAtRuntimeTest.java b/test/jdk/com/alibaba/wisp/thread/DisableThreadAsWispAtRuntimeTest.java
new file mode 100644
index 00000000000..b1eeb4e0080
--- /dev/null
+++ b/test/jdk/com/alibaba/wisp/thread/DisableThreadAsWispAtRuntimeTest.java
@@ -0,0 +1,79 @@
+/*
+ * @test
+ * @summary test to turn on/off shiftThreadModel on demand
+ * @modules java.base/jdk.internal.misc
+ * @modules java.base/com.alibaba.wisp.engine:+open
+ * @library /lib/testlibrary
+ * @run main/othervm -XX:+EnableCoroutine -XX:+UseWispMonitor -Dcom.alibaba.wisp.transparentWispSwitch=true DisableThreadAsWispAtRuntimeTest
+ * @run main/othervm -XX:+EnableCoroutine -XX:+UseWispMonitor -Dcom.alibaba.wisp.transparentWispSwitch=true -Dcom.alibaba.wisp.version=2 DisableThreadAsWispAtRuntimeTest
+*/
+
+import com.alibaba.wisp.engine.WispEngine;
+import com.alibaba.wisp.engine.WispWorkerContainer;
+import jdk.internal.access.SharedSecrets;
+
+import java.io.File;
+import java.io.FileWriter;
+import java.lang.reflect.Field;
+import java.lang.reflect.Method;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+
+import static jdk.testlibrary.Asserts.assertEQ;
+
+public class DisableThreadAsWispAtRuntimeTest {
+
+    private static int wisp_version;
+
+    public static void main(String[] args) throws Exception {
+        File f = File.createTempFile("wisp-", ".config");
+        System.setProperty("com.alibaba.wisp.config", f.getAbsolutePath());
+        f.deleteOnExit();
+        FileWriter writer = new FileWriter(f);
+        writer.write("com.alibaba.wisp.biz.manage=DisableThreadAsWispAtRuntimeTest::main\n");
+        writer.close();
+
+        // WispBizSniffer has already been loaded;
+        // reload WispBizSniffer's config from file.
+        Method m = Class.forName("com.alibaba.wisp.engine.WispConfiguration").getDeclaredMethod("loadBizConfig");
+        m.setAccessible(true);
+        m.invoke(null);
+
+        Class wispClazz = Class.forName("com.alibaba.wisp.engine.WispConfiguration");
+        Field field = wispClazz.getDeclaredField("WISP_VERSION");
+        field.setAccessible(true);
+        wisp_version = field.getInt(null);
+
+        ExecutorService es = Executors.newFixedThreadPool(100);
+
+        for (int i = 0; i < 1000; i++) {
+            boolean on = i % 2 == 0;
+            switchShift(on);
+            for (int j = 0; j < 50; j++) {
+                assertEQ(on, es.submit(DisableThreadAsWispAtRuntimeTest::inCoroutine).get());
+            }
+        }
+
+        es.shutdown();
+    }
+
+    private static void switchShift(boolean val) {
+        try {
+            Class wispClazz = Class.forName("com.alibaba.wisp.engine.WispEngine");
+            Field field = wispClazz.getDeclaredField("shiftThreadModel");
+            field.setAccessible(true);
+            field.setBoolean(null /*static field*/, val);
+        } catch (ReflectiveOperationException e) {
+            throw new Error(e);
+        }
+        assertEQ(WispEngine.enableThreadAsWisp(), val);
+    }
+
+    private static boolean inCoroutine() {
+        return (wisp_version == 2 || SharedSecrets.getJavaLangAccess().currentThread0()
+                .getName().startsWith(WispWorkerContainer.WISP_THREAD_NAME_PREFIX))
+
+                && !SharedSecrets.getWispEngineAccess().isThreadTask(
+                SharedSecrets.getWispEngineAccess().getCurrentTask());
+    }
+}
diff --git a/test/jdk/com/alibaba/wisp/thread/EngineExecutorTest.java b/test/jdk/com/alibaba/wisp/thread/EngineExecutorTest.java
new file mode 100644
index 00000000000..5fb13f2a312
--- /dev/null
+++ b/test/jdk/com/alibaba/wisp/thread/EngineExecutorTest.java
@@ -0,0 +1,47 @@
+/*
+ * @test
+ * @summary test submit task to engine.
+ * @modules java.base/jdk.internal.misc
+ * @run main/othervm -XX:+EnableCoroutine -Dcom.alibaba.wisp.transparentWispSwitch=true EngineExecutorTest
+ * @run main/othervm -XX:+EnableCoroutine -Dcom.alibaba.wisp.transparentWispSwitch=true -Dcom.alibaba.wisp.version=2 EngineExecutorTest 
+*/
+
+import com.alibaba.wisp.engine.WispEngine;
+import jdk.internal.access.SharedSecrets;
+
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.Executor;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+
+public class EngineExecutorTest {
+    public static void main(String[] args) throws Exception {
+        testExecutor();
+    }
+
+    private static void testExecutor() throws Exception {
+        CountDownLatch latch = new CountDownLatch(1);
+        Thread t = new Thread(() -> {
+            WispEngine.current();
+            latch.countDown();
+            while (true) {
+                try {
+                    Thread.sleep(1000);
+                } catch (InterruptedException e) {
+                    e.printStackTrace();
+                }
+            }
+        });
+
+        t.start();
+        latch.await();
+
+        Executor e = SharedSecrets.getJavaLangAccess().getWispEngine(t);
+        CountDownLatch latch1 = new CountDownLatch(100);
+        for (int i = 0; i < 100; i++) {
+            e.execute(latch1::countDown);
+        }
+
+        latch1.await(1, TimeUnit.SECONDS);
+    }
+}
diff --git a/test/jdk/com/alibaba/wisp/thread/InterruptTest.java b/test/jdk/com/alibaba/wisp/thread/InterruptTest.java
new file mode 100644
index 00000000000..cfecce6afd3
--- /dev/null
+++ b/test/jdk/com/alibaba/wisp/thread/InterruptTest.java
@@ -0,0 +1,78 @@
+/*
+ * @test
+ * @summary test thread.interrupt() of wispTask
+ * @run main/othervm -XX:+EnableCoroutine -Dcom.alibaba.wisp.transparentWispSwitch=true InterruptTest
+ * @run main/othervm -XX:+EnableCoroutine -Dcom.alibaba.wisp.transparentWispSwitch=true -Dcom.alibaba.wisp.version=2 InterruptTest
+ */
+
+import com.alibaba.wisp.engine.WispEngine;
+import com.alibaba.wisp.engine.WispTask;
+
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.concurrent.locks.Condition;
+import java.util.concurrent.locks.Lock;
+import java.util.concurrent.locks.ReentrantLock;
+import java.util.concurrent.CountDownLatch;
+
+public class InterruptTest {
+    public static void main(String[] args) throws Exception {
+        Lock lock = new ReentrantLock();
+        Condition dummy = lock.newCondition();
+        Condition finish = lock.newCondition();
+        Condition threadSet = lock.newCondition();
+        CountDownLatch finishLatch = new CountDownLatch(1);
+
+        AtomicBoolean interrupted = new AtomicBoolean(false);
+        AtomicReference thrd = new AtomicReference<>();
+
+        WispEngine.dispatch(() -> {
+            thrd.set(Thread.currentThread());
+            lock.lock();
+            finishLatch.countDown();
+            threadSet.signal();
+            try {
+                try {
+                    dummy.await(1, TimeUnit.SECONDS);
+                    throw new Error("Exception not happened");
+                } catch (InterruptedException e) {
+                    interrupted.set(true);
+                }
+            } finally {
+                finish.signal();
+                lock.unlock();
+            }
+        });
+
+        finishLatch.await();
+        lock.lock();
+        try {
+            if (thrd.get() == null && !threadSet.await(1, TimeUnit.SECONDS))
+                throw new Error("wait threadSet");
+        } finally {
+            lock.unlock();
+        }
+
+        if (thrd.get() == null)
+            throw new Error("thread not set");
+
+        try {
+            Thread.sleep(100);
+        } catch (InterruptedException e) {
+        }
+        thrd.get().interrupt();
+
+        lock.lock();
+        try {
+            if (!finish.await(1, TimeUnit.SECONDS))
+                throw new Error("wait finish");
+        } finally {
+            lock.unlock();
+        }
+
+        if (!interrupted.get())
+            throw new Error("InterruptedException not happened");
+
+    }
+}
diff --git a/test/jdk/com/alibaba/wisp/thread/InterruptedSleepTest.java b/test/jdk/com/alibaba/wisp/thread/InterruptedSleepTest.java
new file mode 100644
index 00000000000..68babde1f9f
--- /dev/null
+++ b/test/jdk/com/alibaba/wisp/thread/InterruptedSleepTest.java
@@ -0,0 +1,36 @@
+/*
+ * @test
+ * @summary test InterruptedException was thrown by sleep()
+ * @library /lib/testlibrary
+ * @run main/othervm -XX:+EnableCoroutine -Dcom.alibaba.wisp.transparentWispSwitch=true InterruptedSleepTest
+ * @run main/othervm -XX:+EnableCoroutine -Dcom.alibaba.wisp.transparentWispSwitch=true -Dcom.alibaba.wisp.version=2 InterruptedSleepTest
+*/
+
+import com.alibaba.wisp.engine.WispEngine;
+
+import static jdk.testlibrary.Asserts.assertFalse;
+import static jdk.testlibrary.Asserts.assertLessThan;
+import static jdk.testlibrary.Asserts.assertTrue;
+
+public class InterruptedSleepTest {
+    public static void main(String[] args) {
+        Thread mainCoro = Thread.currentThread();
+        WispEngine.dispatch(() -> {
+            try {
+                Thread.sleep(100);
+            } catch (InterruptedException e) {
+            }
+            mainCoro.interrupt();
+        });
+        long start = System.currentTimeMillis();
+        boolean ie = false;
+        try {
+            Thread.sleep(1000L);
+        } catch (InterruptedException e) {
+            ie = true;
+        }
+        assertLessThan((int) (System.currentTimeMillis() - start), 1000);
+        assertTrue(ie);
+        assertFalse(mainCoro.isInterrupted());
+    }
+}
diff --git a/test/jdk/com/alibaba/wisp/thread/IsAliveTest.java b/test/jdk/com/alibaba/wisp/thread/IsAliveTest.java
new file mode 100644
index 00000000000..6b275cfbb92
--- /dev/null
+++ b/test/jdk/com/alibaba/wisp/thread/IsAliveTest.java
@@ -0,0 +1,58 @@
+/*
+ * @test
+ * @summary test thread.isAlive() of wispTask
+ * @modules java.base/com.alibaba.wisp.engine:+open
+ * @library /lib/testlibrary
+ * @run main/othervm -XX:+EnableCoroutine -Dcom.alibaba.wisp.transparentWispSwitch=true -Dcom.alibaba.wisp.enableThreadAsWisp=true IsAliveTest
+ * @run main/othervm -XX:+EnableCoroutine -Dcom.alibaba.wisp.transparentWispSwitch=true -Dcom.alibaba.wisp.enableThreadAsWisp=true -Dcom.alibaba.wisp.version=2 IsAliveTest
+ *
+ */
+
+import com.alibaba.wisp.engine.WispEngine;
+
+import java.lang.reflect.Field;
+import java.util.Collections;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicReference;
+
+import static jdk.testlibrary.Asserts.assertFalse;
+import static jdk.testlibrary.Asserts.assertTrue;
+
+
+public class IsAliveTest {
+    public static void main(String[] args) throws Exception {
+        testCurrentDispatch();
+        testContainerDispatch();
+    }
+
+    private static void testCurrentDispatch() {
+        AtomicBoolean isAlive = new AtomicBoolean();
+        AtomicReference t = new AtomicReference<>();
+        final CountDownLatch cond = new CountDownLatch(1);
+
+        WispEngine.dispatch(() -> {
+            t.set(Thread.currentThread());
+            isAlive.set(t.get().isAlive());
+            cond.countDown();
+        });
+        try { 
+            cond.await();
+        } catch (InterruptedException e) {
+            e.printStackTrace();
+        }
+        assertTrue(isAlive.get());
+        assertFalse(t.get().isAlive());
+    }
+
+    private static void testContainerDispatch() throws Exception {
+        Field f = Class.forName("com.alibaba.wisp.engine.WispConfiguration").getDeclaredField("PUT_TO_MANAGED_THREAD_STACK_LIST");
+        f.setAccessible(true);
+        f.set(null, Collections.singletonList("IsAliveTest::testContainerDispatch"));
+        AtomicBoolean finish = new AtomicBoolean();
+        Thread t = new Thread(() -> finish.set(true));
+        t.start();
+
+        assertTrue(t.isAlive() || finish.get());
+    }
+}
diff --git a/test/jdk/com/alibaba/wisp/thread/SubmittedTaskLimitTest.java b/test/jdk/com/alibaba/wisp/thread/SubmittedTaskLimitTest.java
new file mode 100644
index 00000000000..f1c09e31a76
--- /dev/null
+++ b/test/jdk/com/alibaba/wisp/thread/SubmittedTaskLimitTest.java
@@ -0,0 +1,62 @@
+/*
+ * @test
+ * @summary Test to verify threshold setting for submitted wisp tasks.
+ * @modules java.base/com.alibaba.wisp.engine:+open
+ * @library /lib/testlibrary
+ * @run main/othervm -XX:+EnableCoroutine -Dcom.alibaba.wisp.transparentWispSwitch=true SubmittedTaskLimitTest
+ */
+
+import com.alibaba.wisp.engine.WispEngine;
+
+import java.lang.reflect.Field;
+import java.util.Arrays;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.stream.Collectors;
+
+import static jdk.testlibrary.Asserts.assertGT;
+import static jdk.testlibrary.Asserts.assertGTE;
+import static jdk.testlibrary.Asserts.assertTrue;
+
+
+public class SubmittedTaskLimitTest {
+    private static final int SLEEP_TIME = 400;
+
+    public static void main(String[] args) throws Exception {
+        AtomicReference engine = new AtomicReference<>();
+        CountDownLatch l = new CountDownLatch(1);
+        new Thread(() -> {
+            engine.set(WispEngine.current());
+            l.countDown();
+            try {
+                Thread.sleep(100);
+            } catch (InterruptedException e) {
+                e.printStackTrace();
+            }
+        }).start();
+
+        l.await();
+
+        Field f = Class.forName("com.alibaba.wisp.engine.WispConfiguration").getDeclaredField("EXTERNAL_SUBMIT_THRESHOLD");
+        f.setAccessible(true);
+        final int SUBMIT_THRESHOLD = f.getInt(null);
+
+        long start = System.currentTimeMillis();
+        for (int i = 0; i < SUBMIT_THRESHOLD; i++) {
+            engine.get().execute(() -> {
+                try {
+                    Thread.sleep(SLEEP_TIME);
+                } catch (InterruptedException e) {
+                    e.printStackTrace();
+                }
+            });
+        }
+
+        CountDownLatch l2 = new CountDownLatch(1);
+        //this task will get scheduled at least one of above submitted task terminates
+        engine.get().execute(l2::countDown);
+        l2.await();
+
+        assertGTE(System.currentTimeMillis() - start, 400L);
+    }
+}
diff --git a/test/jdk/com/alibaba/wisp/thread/ThreadAsWispTest.java b/test/jdk/com/alibaba/wisp/thread/ThreadAsWispTest.java
new file mode 100644
index 00000000000..df2c222aa65
--- /dev/null
+++ b/test/jdk/com/alibaba/wisp/thread/ThreadAsWispTest.java
@@ -0,0 +1,227 @@
+/*
+ * @test
+ * @summary test dispatching thread into our managed workers
+ * @modules java.base/jdk.internal.misc
+ * @modules java.base/com.alibaba.wisp.engine:+open
+ * @library /lib/testlibrary
+ * @run main/othervm -XX:+EnableCoroutine -XX:+UseWispMonitor -Dcom.alibaba.wisp.transparentWispSwitch=true -Dcom.alibaba.threadPoolLimit=true -Dcom.alibaba.wisp.enableThreadAsWisp=true ThreadAsWispTest
+ * @run main/othervm -XX:+EnableCoroutine -XX:+UseWispMonitor -Dcom.alibaba.wisp.transparentWispSwitch=true -Dcom.alibaba.threadPoolLimit=true -Dcom.alibaba.wisp.enableThreadAsWisp=true -Dcom.alibaba.wisp.version=2 -XX:ActiveProcessorCount=4 ThreadAsWispTest
+*/
+
+
+import com.alibaba.wisp.engine.WispEngine;
+import com.alibaba.wisp.engine.WispWorkerContainer;
+import jdk.internal.access.SharedSecrets;
+
+import java.io.File;
+import java.io.FileWriter;
+import java.lang.reflect.Field;
+import java.lang.reflect.Method;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.*;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.stream.Collectors;
+import java.util.concurrent.locks.Condition;
+import java.util.concurrent.locks.Lock;
+import java.util.concurrent.locks.ReentrantLock;
+import java.util.function.BiConsumer;
+import java.util.stream.IntStream;
+
+import static jdk.testlibrary.Asserts.assertEQ;
+import static jdk.testlibrary.Asserts.assertTrue;
+
+public class ThreadAsWispTest {
+    static Thread mainThread;
+
+    private static Lock lock = new ReentrantLock();
+    private static Condition cond = lock.newCondition();
+    private static BiConsumer consumer = (es, counter) -> {
+        es.execute(() -> {
+            try {
+                Thread.sleep(1_000L);
+                counter.countDown();
+            } catch (InterruptedException e) {
+                e.printStackTrace();
+            }
+        });
+    };
+    private static Executor executor;
+    private static CountDownLatch end;
+    private static int wisp_version;
+
+    public static void main(String[] args) throws Exception {
+        // create real java threads at test beginning
+        final CountDownLatch begin = new CountDownLatch(5);
+        IntStream.range(0, 5).forEach(i -> new Thread(() -> {
+            try {
+                lock.lock();
+                begin.countDown();
+                cond.await();
+                consumer.accept(executor, end);
+            } catch (Throwable e) {
+                e.printStackTrace();
+            } finally {
+                lock.unlock();
+            }
+        }, "Thread ~ " + i).start());
+
+        begin.await();
+
+        File f = File.createTempFile("wisp-", ".config");
+        System.setProperty("com.alibaba.wisp.config", f.getAbsolutePath());
+        f.deleteOnExit();
+        FileWriter writer = new FileWriter(f);
+        writer.write("com.alibaba.wisp.biz.manage=ThreadAsWispTest::main\n");
+        writer.write("com.alibaba.wisp.biz.current=ThreadAsWispTest::main\n");
+        writer.write("com.alibaba.wisp.biz.black=ThreadAsWispTest::shouldNotShift\n");
+
+        writer.close();
+
+        // WispBizSniffer has already been loaded;
+        // reload WispBizSniffer's config from file.
+        Method m = Class.forName("com.alibaba.wisp.engine.WispConfiguration").getDeclaredMethod("loadBizConfig");
+        m.setAccessible(true);
+        m.invoke(null);
+
+        Class wispClazz = Class.forName("com.alibaba.wisp.engine.WispConfiguration");
+        Field field = wispClazz.getDeclaredField("WISP_VERSION");
+        field.setAccessible(true);
+        wisp_version = field.getInt(null);
+
+        AtomicBoolean success = new AtomicBoolean(true);
+        CountDownLatch done = new CountDownLatch(1);
+        AtomicReference t = new AtomicReference<>();
+
+        // test Thread.start()
+        t.set(new Thread(() -> {
+            Thread realThread = SharedSecrets.getJavaLangAccess().currentThread0();
+            boolean value = t.get() == Thread.currentThread()
+                    && !SharedSecrets.getWispEngineAccess().isThreadTask(
+                    SharedSecrets.getWispEngineAccess().getCurrentTask())
+                    && Thread.currentThread().getName().startsWith("new_thread");
+            success.set(value);
+            done.countDown();
+
+        }, "new_thread"));
+
+        t.get().start();
+        done.await();
+
+        assertTrue(success.get());
+
+        // test tread pool
+        CountDownLatch done1 = new CountDownLatch(1);
+        final String NAME = "ShiftThreadModelTest-pool";
+        ExecutorService es = Executors.newCachedThreadPool(r -> {
+            Thread t1 = new Thread(r);
+            t1.setName(NAME);
+            return t1;
+        });
+        mainThread = SharedSecrets.getJavaLangAccess().currentThread0();
+        es.execute(() -> {
+            Thread realThread = SharedSecrets.getJavaLangAccess().currentThread0();
+            if (wisp_version == 2) {
+                success.set(realThread != mainThread
+                    && Thread.currentThread() != realThread
+                    && Thread.currentThread().getClass().getSimpleName().contains("WispThreadWrapper")
+                    && Thread.currentThread().getName().startsWith(NAME)
+                    && !SharedSecrets.getWispEngineAccess().isThreadTask(
+                        SharedSecrets.getWispEngineAccess().getCurrentTask())
+                );
+            } else {
+                success.set(realThread == mainThread
+                    && Thread.currentThread() != realThread
+                    && Thread.currentThread().getClass().getSimpleName().contains("WispThreadWrapper")
+                    && Thread.currentThread().getName().startsWith(NAME)
+                    && !SharedSecrets.getWispEngineAccess().isThreadTask(
+                        SharedSecrets.getWispEngineAccess().getCurrentTask())
+                );
+            }
+            done1.countDown();
+        });
+        done1.await();
+        assertTrue(success.get());
+
+        shouldNotShift();
+
+        // test thread pool wisp limit
+        testThreadPoolLimit(new LinkedBlockingQueue(), 3, 5, false);
+        testThreadPoolLimit(new LinkedBlockingQueue(3), 3, 10, true);
+        testThreadPoolLimit(new LinkedBlockingQueue(3), 3, 6, false);
+        System.out.println("LinkedBlockingQueue ok!");
+        testThreadPoolLimit(new LinkedBlockingDeque(), 3, 1, false);
+        testThreadPoolLimit(new LinkedBlockingDeque(3), 3, 10, true);
+        testThreadPoolLimit(new LinkedBlockingDeque(3), 3, 6, false);
+        System.out.println("LinkedBlockingDeque ok!");
+        testThreadPoolLimit(new ArrayBlockingQueue(3), 3, 10, true);
+        testThreadPoolLimit(new ArrayBlockingQueue(3), 3, 6, false);
+        System.out.println("ArrayBlockingQueue ok!");
+        testThreadPoolLimit(new SynchronousQueue(), 3, 6, true);
+        testThreadPoolLimit(new SynchronousQueue(), 3, 3, false);
+        System.out.println("SynchronousQueue ok!");
+        testThreadPoolLimit(new DelayQueue(), 1, 3, false);
+        System.out.println("DelayQueue ok!");
+        testThreadPoolLimit(new LinkedTransferQueue(), 1, 3, false);
+        testThreadPoolLimit(new LinkedTransferQueue(), 3, 3, false);
+        System.out.println("LinkedTransferQueue ok!");
+        testThreadPoolLimit(new PriorityBlockingQueue(), 1, 3, false);
+        System.out.println("PriorityBlockingQueue ok!");
+
+        // multi-thread test
+        RejectedExecutionHandler rejectHandler = (r, executor) -> {
+            success.set(true);
+        };
+        executor = new ThreadPoolExecutor(1, 1, Long.MAX_VALUE, TimeUnit.SECONDS,
+                new ArrayBlockingQueue(1), rejectHandler);
+        success.set(false);
+        end = new CountDownLatch(1);
+        lock.lock();
+        cond.signalAll();
+        lock.unlock();
+        end.await();
+        assertTrue(success.get());
+    }
+
+    private static void testThreadPoolLimit(BlockingQueue queue, int maxPoolSize, int submitCount, boolean expect) throws InterruptedException {
+        AtomicBoolean success = new AtomicBoolean(false);
+        RejectedExecutionHandler rejectHandler = (r, executor) -> {
+            success.set(true);
+        };
+        executor = new ThreadPoolExecutor(1, maxPoolSize, Long.MAX_VALUE, TimeUnit.SECONDS,
+                queue, rejectHandler);
+        end = new CountDownLatch(1);
+        IntStream.range(0, submitCount).forEach(i -> {
+            consumer.accept(executor, end);
+        });
+        end.await();
+        assertEQ(success.get(), expect);
+    }
+
+    // test black list
+    static private void shouldNotShift() throws Exception {
+        // even we're in the main() method,  the thread model should not be changed,
+        // because we're in the black list
+        ExecutorService es = Executors.newCachedThreadPool();
+        AtomicBoolean success = new AtomicBoolean(true);
+        CountDownLatch done = new CountDownLatch(1);
+        es.execute(() -> {
+            Thread realThread = SharedSecrets.getJavaLangAccess().currentThread0();
+            success.set(realThread != mainThread &&
+                    !WispWorkerContainer.INSTANCE.getWorkers().stream().map(e -> {
+                        try {
+                            Field f = WispEngine.class.getDeclaredField("thread");
+                            f.setAccessible(true);
+                            return f.get(e);
+                        } catch (Exception e1) {
+                            throw new Error();
+                        }
+                    }).collect(Collectors.toList()).contains(realThread));
+            done.countDown();
+        });
+        done.await();
+        assertTrue(success.get());
+    }
+}
diff --git a/test/jdk/com/alibaba/wisp/thread/ThrowErrorTest.java b/test/jdk/com/alibaba/wisp/thread/ThrowErrorTest.java
new file mode 100644
index 00000000000..373ceb31a8e
--- /dev/null
+++ b/test/jdk/com/alibaba/wisp/thread/ThrowErrorTest.java
@@ -0,0 +1,28 @@
+/*
+ * @test
+ * @summary test coroutine throw Error
+ * @library /lib/testlibrary
+ * @run main/othervm -XX:+EnableCoroutine -Dcom.alibaba.wisp.transparentWispSwitch=true ThrowErrorTest
+ * @run main/othervm -XX:+EnableCoroutine -Dcom.alibaba.wisp.transparentWispSwitch=true -Dcom.alibaba.wisp.version=2 ThrowErrorTest
+*/
+
+import com.alibaba.wisp.engine.WispEngine;
+
+import static jdk.testlibrary.Asserts.assertTrue;
+
+public class ThrowErrorTest {
+    public static void main(String[] args) {
+        WispEngine.dispatch(() -> {
+            throw new Error();
+        });
+
+        boolean[] executed = new boolean[]{false};
+
+        WispEngine.dispatch(() -> executed[0] = true);
+        try {
+            Thread.sleep(100);
+        } catch (InterruptedException e) {
+        }
+        assertTrue(executed[0]);
+    }
+}
diff --git a/test/jdk/com/alibaba/wisp/thread/TimeSliceTest.java b/test/jdk/com/alibaba/wisp/thread/TimeSliceTest.java
new file mode 100644
index 00000000000..d2c38f79e20
--- /dev/null
+++ b/test/jdk/com/alibaba/wisp/thread/TimeSliceTest.java
@@ -0,0 +1,45 @@
+/*
+ * @test
+ * @summary test wisp time slice
+ * @library /lib/testlibrary
+ * @run main/othervm -XX:+EnableCoroutine  -XX:+EnableCoroutineTimeSlice -Dcom.alibaba.wisp.transparentWispSwitch=true -Dcom.alibaba.wisp.enableHandOff=true -Dcom.alibaba.wisp.sysmonTickUs=10000 -XX:-Inline -Dcom.alibaba.wisp.version=2 TimeSliceTest
+*/
+
+import com.alibaba.wisp.engine.WispEngine;
+
+import java.util.Date;
+import java.util.concurrent.atomic.AtomicReference;
+
+import static jdk.testlibrary.Asserts.assertTrue;
+
+public class TimeSliceTest {
+    public static void main(String[] args) throws Exception {
+        AtomicReference engine = new AtomicReference<>();
+        Thread th = new Thread(() -> {
+            WispEngine.dispatch(() -> {});  // register engine
+            engine.set(WispEngine.current());
+            while (true) {
+                foo();
+            }
+        }, "engine_thread");
+        th.setDaemon(true);
+        th.start();
+
+        while (engine.get() == null) {
+        }
+
+        boolean[] b = new boolean[]{false};
+
+        engine.get().execute(() -> b[0] = true);
+
+        Thread.sleep(1000);
+
+        assertTrue(b[0]);
+    }
+
+    static int n;
+
+    private static void foo() {
+       n = new Date().toString().hashCode();
+    }
+}
diff --git a/test/jdk/com/alibaba/wisp/thread/YieldTest.java b/test/jdk/com/alibaba/wisp/thread/YieldTest.java
new file mode 100644
index 00000000000..7a2ec67455e
--- /dev/null
+++ b/test/jdk/com/alibaba/wisp/thread/YieldTest.java
@@ -0,0 +1,29 @@
+/*
+ * @test
+ * @summary test yield()
+ * @library /lib/testlibrary
+ * @run main/othervm -XX:+EnableCoroutine -Dcom.alibaba.wisp.transparentWispSwitch=true YieldTest
+ */
+
+import com.alibaba.wisp.engine.WispEngine;
+
+import static jdk.testlibrary.Asserts.*;
+
+public class YieldTest {
+    private static int i = 0;
+    public static void main(String[] args) {
+        WispEngine.dispatch(() -> {
+            assertEQ(i++, 0);
+            Thread.yield();
+            assertEQ(i++, 2);
+            Thread.yield();
+            assertEQ(i++, 4);
+            Thread.yield();
+        });
+        assertEQ(i++, 1);
+        Thread.yield();
+        assertEQ(i++, 3);
+        Thread.yield();
+        assertEQ(i++, 5);
+    }
+}
diff --git a/test/jdk/com/alibaba/wisp/timer/DaemonThreadGroupTest.java b/test/jdk/com/alibaba/wisp/timer/DaemonThreadGroupTest.java
new file mode 100644
index 00000000000..82cf959998f
--- /dev/null
+++ b/test/jdk/com/alibaba/wisp/timer/DaemonThreadGroupTest.java
@@ -0,0 +1,63 @@
+/*
+ * @test
+ * @library /lib/testlibrary
+ * @summary Test Daemon Thread Group implementation
+ * @modules java.base/jdk.internal.misc
+ * @modules java.base/com.alibaba.wisp.engine:+open
+ * @run main/othervm -XX:+EnableCoroutine -Dcom.alibaba.wisp.version=2 DaemonThreadGroupTest
+*/
+
+
+import com.alibaba.wisp.engine.WispEngine;
+import com.alibaba.wisp.engine.WispTask;
+import com.alibaba.wisp.engine.WispPoller;
+import jdk.internal.access.SharedSecrets;
+import jdk.internal.access.WispEngineAccess;
+
+import java.lang.reflect.Field;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicReference;
+
+import static jdk.testlibrary.Asserts.assertTrue;
+
+
+/**
+ * test the Daemon Thread Group implementation
+ */
+public class DaemonThreadGroupTest {
+    public static void main(String... arg) throws Exception {
+
+        WispEngineAccess access = SharedSecrets.getWispEngineAccess();
+        access.eventLoop();
+
+        AtomicReference task1 = new AtomicReference<>();
+        AtomicBoolean doUnpark = new AtomicBoolean(false);
+        AtomicBoolean hasError = new AtomicBoolean(false);
+
+        Field f = Class.forName("com.alibaba.wisp.engine.WispPoller").getDeclaredField("thread");
+        f.setAccessible(true);
+        Thread t = (Thread) f.get(WispPoller.INSTANCE);
+
+        f = Class.forName("com.alibaba.wisp.engine.WispEngine").getDeclaredField("daemonThreadGroup");
+        f.setAccessible(true);
+        ThreadGroup threadGroup = (ThreadGroup) f.get(null);
+
+        System.out.println(threadGroup.getName());
+
+        assertTrue(t.getThreadGroup() == threadGroup, "the thread isn't in daemonThreadGroup");
+
+        WispEngine.dispatch(() -> {
+            task1.set(access.getCurrentTask());
+            access.park(Long.MAX_VALUE);
+            // if timeout is negative(< now()), this task is selected in doSchedule
+            // and park returns immediately
+            hasError.set(!doUnpark.get()); // should not reach here before doing unpark
+        });
+
+        access.sleep(100); // switch task
+        // let task exit
+        doUnpark.set(true);
+        access.unpark(task1.get());
+        assertTrue(!hasError.get(), "hasError.get() should be false.");
+    }
+}
diff --git a/test/jdk/com/alibaba/wisp/timer/OverflowTest.java b/test/jdk/com/alibaba/wisp/timer/OverflowTest.java
new file mode 100644
index 00000000000..088c93debcf
--- /dev/null
+++ b/test/jdk/com/alibaba/wisp/timer/OverflowTest.java
@@ -0,0 +1,49 @@
+/*
+ * @test
+ * @library /lib/testlibrary
+ * @summary Test timer implementation
+ * @modules java.base/jdk.internal.misc
+ * @run main/othervm -XX:+EnableCoroutine OverflowTest
+ * @run main/othervm -XX:+EnableCoroutine -Dcom.alibaba.wisp.version=2 OverflowTest
+*/
+
+
+import com.alibaba.wisp.engine.WispEngine;
+import com.alibaba.wisp.engine.WispTask;
+import jdk.internal.access.SharedSecrets;
+import jdk.internal.access.WispEngineAccess;
+
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicReference;
+
+import static jdk.testlibrary.Asserts.assertTrue;
+
+
+/**
+ * test the time out implementation
+ */
+public class OverflowTest {
+    public static void main(String... arg) throws Exception {
+
+        WispEngineAccess access = SharedSecrets.getWispEngineAccess();
+        access.eventLoop();
+
+        AtomicReference task1 = new AtomicReference<>();
+        AtomicBoolean doUnpark = new AtomicBoolean(false);
+        AtomicBoolean hasError = new AtomicBoolean(false);
+
+        WispEngine.dispatch(() -> {
+            task1.set(access.getCurrentTask());
+            access.park(Long.MAX_VALUE);
+            // if timeout is negative(< now()), this task is selected in doSchedule
+            // and park returns immediately
+            hasError.set(!doUnpark.get()); // should not reach here before doing unpark
+        });
+
+        access.sleep(100); // switch task
+        // let task exit
+        doUnpark.set(true);
+        access.unpark(task1.get());
+        assertTrue(!hasError.get(), "hasError.get() should be false.");
+    }
+}
diff --git a/test/jdk/com/alibaba/wisp/timer/PriorityQueueSortTest.java b/test/jdk/com/alibaba/wisp/timer/PriorityQueueSortTest.java
new file mode 100644
index 00000000000..0a4290cb922
--- /dev/null
+++ b/test/jdk/com/alibaba/wisp/timer/PriorityQueueSortTest.java
@@ -0,0 +1,131 @@
+/*
+ * @test
+ * @library /lib/testlibrary
+ * @summary Test TimeOut.Queue's offer and remove function, make sure it's consistent with the behavior of the jdk's priority queue
+ * @modules java.base/com.alibaba.wisp.engine:+open
+ * @run main/othervm PriorityQueueSortTest
+ */
+
+import com.alibaba.wisp.engine.TimeOut;
+
+import java.lang.reflect.Constructor;
+import java.lang.reflect.Field;
+import java.lang.reflect.Method;
+import java.util.*;
+
+
+import static jdk.testlibrary.Asserts.assertTrue;
+
+public class PriorityQueueSortTest {
+
+	static Class classQ;
+	static Class classT;
+
+	static Method poll = null;
+	static Method offer = null;
+
+	static TimeOut ILLEGAL_FLAG = new TimeOut(null, 1, false);
+
+	static class MyComparator implements Comparator {
+		public int compare(TimeOut x, TimeOut y) {
+			return Long.compare(getDeadNano(x), getDeadNano(y));
+		}
+	}
+
+	static Long getDeadNano(TimeOut t) {
+		try {
+			Field deadline = TimeOut.class.getDeclaredField("deadlineNano");
+			deadline.setAccessible(true);
+			return (Long) deadline.get(t);
+		} catch (Exception e) {
+			throw new Error(e);
+		}
+	}
+
+	static Object getTimerQueue() {
+		try {
+			classQ = Class.forName("com.alibaba.wisp.engine.TimeOut$TimerManager$Queue");
+			classT = Class.forName("com.alibaba.wisp.engine.TimeOut$TimerManager");
+			Constructor c1 = classQ.getDeclaredConstructor(classT);
+			c1.setAccessible(true);
+			Constructor c2 = classT.getDeclaredConstructor();
+			c2.setAccessible(true);
+			poll = classQ.getDeclaredMethod("poll");
+			poll.setAccessible(true);
+			offer = classQ.getDeclaredMethod("offer", TimeOut.class);
+			offer.setAccessible(true);
+			return c1.newInstance(c2.newInstance());
+		} catch (Exception e) {
+			e.printStackTrace();
+			return null;
+		}
+	}
+
+	static boolean add(Object pq, TimeOut timeOut) {
+		try {
+			offer.invoke(pq, timeOut);
+			return true;
+		} catch (Exception e) {
+			e.printStackTrace();
+			return false;
+		}
+	}
+
+	static TimeOut poll(Object pq) {
+		try {
+			return (TimeOut)poll.invoke(pq);
+		} catch (Exception e) {
+			e.printStackTrace();
+			return ILLEGAL_FLAG;
+		}
+	}
+
+	public static void main(String[] args) {
+		int n = 10000;
+
+		List sorted = new ArrayList<>(n);
+		for (int i = 0; i < n; i++)
+			sorted.add(new Long(i));
+		List shuffled = new ArrayList<>(sorted);
+		Collections.shuffle(shuffled);
+
+		Object pq = getTimerQueue();
+
+
+		for (Iterator i = shuffled.iterator(); i.hasNext(); )
+			add(pq, new TimeOut(null, i.next(), false));
+
+		List recons = new ArrayList<>();
+
+		while (true) {
+			TimeOut t = poll(pq);
+			if (t == null) {
+				break;
+			}
+			recons.add(getDeadNano(t));
+		}
+		assertTrue(recons.equals(sorted), "Sort failed");
+
+		for (Long val : recons) {
+			add(pq, new TimeOut(null, val, false));
+		}
+		recons.clear();
+		while(true){
+			TimeOut timeOut = poll(pq);
+			if (timeOut == null) {
+				break;
+			}
+			if(getDeadNano(timeOut)  % 2 == 1) {
+				recons.add(getDeadNano(timeOut));
+			}
+		}
+
+		Collections.sort(recons);
+
+		for (Iterator i = sorted.iterator(); i.hasNext(); )
+			if ((i.next().intValue() % 2) != 1)
+				i.remove();
+
+		assertTrue(recons.equals(sorted), "Odd Sort failed");
+	}
+}
diff --git a/test/jdk/com/alibaba/wisp/timer/SleepRPCTest.java b/test/jdk/com/alibaba/wisp/timer/SleepRPCTest.java
new file mode 100644
index 00000000000..85cabc40a85
--- /dev/null
+++ b/test/jdk/com/alibaba/wisp/timer/SleepRPCTest.java
@@ -0,0 +1,50 @@
+/*
+ * @test
+ * @summary test use sleep in RPC senorina
+ * @library /lib/testlibrary
+ * @run main/othervm -XX:+EnableCoroutine -XX:+UseWispMonitor -Dcom.alibaba.wisp.transparentWispSwitch=true  -Dcom.alibaba.wisp.enableThreadAsWisp=true SleepRPCTest
+ * @run main/othervm -XX:+EnableCoroutine -XX:+UseWispMonitor -Dcom.alibaba.wisp.transparentWispSwitch=true  -Dcom.alibaba.wisp.enableThreadAsWisp=true -Dcom.alibaba.wisp.version=2 SleepRPCTest
+ */
+
+import com.alibaba.wisp.engine.WispEngine;
+
+import java.util.List;
+import java.util.Random;
+import java.util.concurrent.Future;
+import java.util.concurrent.FutureTask;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
+
+import static jdk.testlibrary.Asserts.assertTrue;
+
+public class SleepRPCTest {
+    public static void main(String[] args) {
+
+        List> rpcs = IntStream.range(0, 50).mapToObj(i -> {
+            AtomicBoolean done = new AtomicBoolean(false);
+            FutureTask ft = new FutureTask<>(() -> {
+                for (int n = 0; !done.get() && n < 40; n++) {
+                    Thread.sleep(5);
+                } // 200 ms timeout
+                return done.get();
+            });
+            WispEngine.dispatch(ft);
+            WispEngine.dispatch(new FutureTask<>(() -> {
+                Thread.sleep(new Random().nextInt(100) + 1);
+                done.set(true);
+                return 0;
+            }));
+
+            return ft;
+        }).collect(Collectors.toList());
+
+        assertTrue(rpcs.stream().allMatch(ft -> {
+            try {
+                return ft.get();
+            } catch (Throwable e) {
+                throw new Error(e);
+            }
+        }));
+    }
+}
diff --git a/test/jdk/com/alibaba/wisp/timer/SleepTest.java b/test/jdk/com/alibaba/wisp/timer/SleepTest.java
new file mode 100644
index 00000000000..197e2e537b1
--- /dev/null
+++ b/test/jdk/com/alibaba/wisp/timer/SleepTest.java
@@ -0,0 +1,42 @@
+/*
+ * @test
+ * @summary test sleep
+ * @library /lib/testlibrary
+ * @run main/othervm -XX:+EnableCoroutine -XX:+UseWispMonitor -Dcom.alibaba.wisp.transparentWispSwitch=true  -Dcom.alibaba.wisp.enableThreadAsWisp=true -Dcom.alibaba.wisp.version=2 SleepTest
+ */
+
+import com.alibaba.wisp.engine.WispEngine;
+
+import java.util.concurrent.FutureTask;
+import java.util.concurrent.TimeUnit;
+import java.util.stream.IntStream;
+
+import static jdk.testlibrary.Asserts.assertTrue;
+
+public class SleepTest {
+    public static void main(String[] args) {
+        assertTrue(IntStream.range(0, 100).parallel().allMatch(ms -> {
+            long start = System.currentTimeMillis();
+            FutureTask ft = new FutureTask<>(() -> {
+                Thread.sleep(ms);
+                return 0;
+            });
+            WispEngine.dispatch(ft);
+            try {
+                ft.get(200, TimeUnit.MILLISECONDS);
+            } catch (Throwable e) {
+                throw new Error(e);
+            }
+
+            long interval = System.currentTimeMillis() - start;
+
+            if (Math.abs(interval - ms) < 100) {
+                return true;
+            } else {
+                System.out.println("ms = " + ms);
+                System.out.println("interval = " + interval);
+                return false;
+            }
+        }));
+    }
+}
diff --git a/test/jdk/com/alibaba/wisp/timer/TimerTest.java b/test/jdk/com/alibaba/wisp/timer/TimerTest.java
new file mode 100644
index 00000000000..e221af92d07
--- /dev/null
+++ b/test/jdk/com/alibaba/wisp/timer/TimerTest.java
@@ -0,0 +1,34 @@
+/*
+ * @test
+ * @summary Test timer implement
+ * @modules java.base/jdk.internal.misc
+ * @run main/othervm -XX:+EnableCoroutine TimerTest
+ * @run main/othervm -XX:+EnableCoroutine -Dcom.alibaba.wisp.version=2 TimerTest
+*/
+
+
+import com.alibaba.wisp.engine.WispEngine;
+import com.alibaba.wisp.engine.WispTask;
+import jdk.internal.access.SharedSecrets;
+
+
+/**
+ * test the time out implement
+ */
+public class TimerTest {
+    public static void main(String... arg) throws Exception {
+        WispEngine.dispatch(() -> {
+            long ts = System.currentTimeMillis();
+            for (int i = 0; i < 10; i++) {
+                SharedSecrets.getWispEngineAccess().sleep(100);
+                long now = System.currentTimeMillis();
+                if (Math.abs(now - ts - 100) > 10)
+                    throw new Error();
+                ts = now;
+            }
+
+        });
+
+        SharedSecrets.getWispEngineAccess().eventLoop();
+    }
+}
diff --git a/test/jdk/com/alibaba/wisp2/AllThreadAsWispTest.java b/test/jdk/com/alibaba/wisp2/AllThreadAsWispTest.java
new file mode 100644
index 00000000000..539bf39bf58
--- /dev/null
+++ b/test/jdk/com/alibaba/wisp2/AllThreadAsWispTest.java
@@ -0,0 +1,50 @@
+/*
+ * @test
+ * @library /lib/testlibrary
+ * @summary convert all thread to wisp
+ * @modules java.base/jdk.internal.misc
+ * @run main/othervm -XX:+EnableCoroutine  -Dcom.alibaba.wisp.transparentWispSwitch=true  -Dcom.alibaba.wisp.version=2 -XX:+UseWispMonitor -Dcom.alibaba.wisp.enableThreadAsWisp=true -Dcom.alibaba.wisp.allThreadAsWisp=true AllThreadAsWispTest
+ */
+
+import jdk.internal.access.SharedSecrets;
+
+import java.util.Timer;
+import java.util.TimerTask;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+
+import static jdk.testlibrary.Asserts.assertTrue;
+
+public class AllThreadAsWispTest {
+    public static void main(String[] args) throws Exception {
+        CountDownLatch done = new CountDownLatch(60);
+
+        Runnable r = () -> {
+            if (SharedSecrets.getJavaLangAccess().currentThread0() != Thread.currentThread()) {
+                done.countDown();
+            }
+        };
+
+        for (int i = 0; i < 10; i++) {
+            Executors.newSingleThreadExecutor().submit(r);
+            Executors.newWorkStealingPool().submit(r);
+            Executors.newScheduledThreadPool(100).submit(r);
+            new Thread(r).start();
+            new Timer().schedule(new TimerTask() {
+                @Override
+                public void run() {
+                    r.run();
+                }
+            }, 0);
+            new Thread() {
+                @Override
+                public void run() {
+                    r.run();
+                }
+            }.start();
+        }
+
+        assertTrue(done.await(10, TimeUnit.SECONDS));
+    }
+}
diff --git a/test/jdk/com/alibaba/wisp2/CtxClassLoaderInheritanceTest.java b/test/jdk/com/alibaba/wisp2/CtxClassLoaderInheritanceTest.java
new file mode 100644
index 00000000000..e5a4b85a774
--- /dev/null
+++ b/test/jdk/com/alibaba/wisp2/CtxClassLoaderInheritanceTest.java
@@ -0,0 +1,29 @@
+/*
+ * @test
+ * @library /lib/testlibrary
+ * @summary test context ClassLoader inherit.
+ * @run main/othervm -XX:+EnableCoroutine  -Dcom.alibaba.wisp.transparentWispSwitch=true  -Dcom.alibaba.wisp.version=2 -XX:+UseWispMonitor -Dcom.alibaba.wisp.enableThreadAsWisp=true -Dcom.alibaba.wisp.allThreadAsWisp=true CtxClassLoaderInheritanceTest
+ */
+
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+
+import static jdk.testlibrary.Asserts.assertEQ;
+
+public class CtxClassLoaderInheritanceTest {
+    public static void main(String[] args) throws Exception {
+        ClassLoader cl = new ClassLoader() {
+        };
+
+        Thread.currentThread().setContextClassLoader(cl);
+        ClassLoader[] childCtxCl = new ClassLoader[1];
+        CountDownLatch done = new CountDownLatch(1);
+        new Thread(() -> {
+            childCtxCl[0] = Thread.currentThread().getContextClassLoader();
+            done.countDown();
+        }).start();
+
+        done.await(1, TimeUnit.SECONDS);
+        assertEQ(cl, childCtxCl[0]);
+    }
+}
diff --git a/test/jdk/com/alibaba/wisp2/DispatchTest.java b/test/jdk/com/alibaba/wisp2/DispatchTest.java
new file mode 100644
index 00000000000..284672a7ca9
--- /dev/null
+++ b/test/jdk/com/alibaba/wisp2/DispatchTest.java
@@ -0,0 +1,27 @@
+/*
+ * @test
+ * @summary basic wisp2
+ * @modules java.base/jdk.internal.misc
+ * @run main/othervm -XX:+EnableCoroutine  -Dcom.alibaba.wisp.transparentWispSwitch=true  -Dcom.alibaba.wisp.version=2 -XX:+UseWispMonitor  DispatchTest
+ */
+
+
+import com.alibaba.wisp.engine.WispEngine;
+import jdk.internal.access.SharedSecrets;
+
+public class DispatchTest {
+    public static void main(String[] args) throws Exception {
+        WispEngine.dispatch(() -> {
+            for (int i = 0; i < 9999999; i++) {
+                try {
+                    Thread.sleep(100);
+                    System.out.println(i + ": " + SharedSecrets.getJavaLangAccess().currentThread0());
+                } catch (InterruptedException e) {
+                    e.printStackTrace();
+                }
+            }
+        });
+        System.out.println("DispatchTest.main");
+        Thread.sleep(1000);
+    }
+}
diff --git a/test/jdk/com/alibaba/wisp2/HandOffTest.java b/test/jdk/com/alibaba/wisp2/HandOffTest.java
new file mode 100644
index 00000000000..8f2dcfd4fb3
--- /dev/null
+++ b/test/jdk/com/alibaba/wisp2/HandOffTest.java
@@ -0,0 +1,50 @@
+/*
+ * @test
+ * @library /lib/testlibrary
+ * @summary test long running or blocking syscall task could be retaken
+ * @run main/othervm -Dcom.alibaba.wisp.carrierEngines=1 -XX:-UseBiasedLocking -XX:+EnableCoroutine -XX:+UseWispMonitor -Dcom.alibaba.wisp.transparentWispSwitch=true  -Dcom.alibaba.wisp.version=2 -Dcom.alibaba.wisp.enableHandOff=true -Dcom.alibaba.wisp.sysmonTickUs=100000 HandOffTest
+ */
+
+import com.alibaba.wisp.engine.WispEngine;
+
+import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.nio.channels.SocketChannel;
+import java.nio.ByteBuffer;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import static jdk.testlibrary.Asserts.assertTrue;
+
+public class HandOffTest {
+    public static void main(String[] args) throws Exception {
+        CountDownLatch cl = new CountDownLatch(10);
+        for (int i = 0; i < 10; i++) {
+            WispEngine.dispatch(() -> {
+                try {
+                    Thread.sleep(1000);
+                } catch (InterruptedException e) {
+                    e.printStackTrace();
+                }
+                cl.countDown();
+            });
+        }
+
+        AtomicBoolean blockingFinish = new AtomicBoolean(false);
+
+        WispEngine.dispatch(() -> {
+            try {
+                SocketChannel ch = SocketChannel.open(new InetSocketAddress("www.example.com", 80));
+                ch.read(ByteBuffer.allocate(4096));
+                blockingFinish.set(true);
+            } catch (IOException e) {
+                e.printStackTrace();
+            }
+        });
+
+
+        assertTrue(cl.await(100, TimeUnit.SECONDS));
+        assertTrue(!blockingFinish.get());
+    }
+}
diff --git a/test/jdk/com/alibaba/wisp2/NioBlockingAcceptTest.java b/test/jdk/com/alibaba/wisp2/NioBlockingAcceptTest.java
new file mode 100644
index 00000000000..ff7389f8956
--- /dev/null
+++ b/test/jdk/com/alibaba/wisp2/NioBlockingAcceptTest.java
@@ -0,0 +1,34 @@
+/*
+ * @test
+ * @library /lib/testlibrary
+ * @summary test nio blocking accept
+ * @run main/othervm -Dcom.alibaba.wisp.carrierEngines=1 -XX:ActiveProcessorCount=1 -XX:+EnableCoroutine  -Dcom.alibaba.wisp.transparentWispSwitch=true  -Dcom.alibaba.wisp.version=2 -XX:+UseWispMonitor -Dcom.alibaba.wisp.enableThreadAsWisp=true -Dcom.alibaba.wisp.allThreadAsWisp=true NioBlockingAcceptTest
+ */
+
+import com.alibaba.wisp.engine.WispEngine;
+
+import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.nio.channels.ServerSocketChannel;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+
+import static jdk.testlibrary.Asserts.assertTrue;
+
+public class NioBlockingAcceptTest {
+    public static void main(String[] args) throws Exception {
+        WispEngine.dispatch(() -> {
+            try {
+                final ServerSocketChannel ssc = ServerSocketChannel.open();
+                ssc.bind(new InetSocketAddress(0));
+                ssc.accept();
+            } catch (IOException e) {
+                e.printStackTrace();
+            }
+        });
+        final CountDownLatch latch = new CountDownLatch(1);
+        WispEngine.dispatch(latch::countDown);
+
+        assertTrue(latch.await(1, TimeUnit.SECONDS));
+    }
+}
diff --git a/test/jdk/com/alibaba/wisp2/ReuseWispTaskAfterThreadJoinTest.java b/test/jdk/com/alibaba/wisp2/ReuseWispTaskAfterThreadJoinTest.java
new file mode 100644
index 00000000000..6c5db785e1b
--- /dev/null
+++ b/test/jdk/com/alibaba/wisp2/ReuseWispTaskAfterThreadJoinTest.java
@@ -0,0 +1,30 @@
+/*
+ * @test
+ * @library /lib/testlibrary
+ * @summary test wisp task reusing after thread.join()
+ * @run main/othervm  -XX:+EnableCoroutine  -Dcom.alibaba.wisp.transparentWispSwitch=true  -Dcom.alibaba.wisp.version=2 -XX:+UseWispMonitor -Dcom.alibaba.wisp.enableThreadAsWisp=true -Dcom.alibaba.wisp.allThreadAsWisp=true  ReuseWispTaskAfterThreadJoinTest
+ */
+
+import com.alibaba.wisp.engine.WispEngine;
+
+public class ReuseWispTaskAfterThreadJoinTest {
+    public static void main(String[] args) throws Exception {
+        Thread t = new Thread(() -> {
+
+
+        });
+        t.start();
+        t.join();
+        // really exited.
+        // reuse the wispTask
+        WispEngine.dispatch(() -> {
+            try {
+                Thread.sleep(200000);
+            } catch (InterruptedException e) {
+                e.printStackTrace();
+            }
+        });
+        Thread.sleep(1000);
+        t.join();
+    }
+}
diff --git a/test/jdk/com/alibaba/wisp2/ThreadJoinTest.java b/test/jdk/com/alibaba/wisp2/ThreadJoinTest.java
new file mode 100644
index 00000000000..d920775e936
--- /dev/null
+++ b/test/jdk/com/alibaba/wisp2/ThreadJoinTest.java
@@ -0,0 +1,37 @@
+/*
+ * @test
+ * @library /lib/testlibrary
+ * @summary test thread.join()
+ * @run main/othervm  -XX:+EnableCoroutine  -Dcom.alibaba.wisp.transparentWispSwitch=true  -Dcom.alibaba.wisp.version=2 -XX:+UseWispMonitor -Dcom.alibaba.wisp.enableThreadAsWisp=true -Dcom.alibaba.wisp.allThreadAsWisp=true ThreadJoinTest
+ */
+
+import com.alibaba.wisp.engine.WispEngine;
+
+public class ThreadJoinTest {
+    public static void main(String[] args) throws Exception {
+        Thread t = new Thread(() -> {
+            try {
+                Thread.sleep(1000);
+            } catch (InterruptedException e) {
+                e.printStackTrace();
+            }
+        });
+        t.start();
+        t.join();
+
+        t = new Thread(() -> {
+
+        });
+        t.start();
+        Thread.sleep(1000);
+        WispEngine.dispatch(() -> {
+            try {
+                Thread.sleep(200000);
+            } catch (InterruptedException e) {
+                e.printStackTrace();
+            }
+        });
+        Thread.sleep(1000);
+        t.join();
+    }
+}
diff --git a/test/jdk/com/alibaba/wisp2/TimedWaitTest.java b/test/jdk/com/alibaba/wisp2/TimedWaitTest.java
new file mode 100644
index 00000000000..70527944cf0
--- /dev/null
+++ b/test/jdk/com/alibaba/wisp2/TimedWaitTest.java
@@ -0,0 +1,30 @@
+/*
+ * @test
+ * @summary test timed Jvm park
+ * @run main/othervm -XX:-UseBiasedLocking -XX:+EnableCoroutine -XX:+UseWispMonitor -Dcom.alibaba.wisp.transparentWispSwitch=true -Dcom.alibaba.wisp.version=2 TimedWaitTest
+ */
+
+import com.alibaba.wisp.engine.WispEngine;
+
+import java.util.concurrent.CountDownLatch;
+
+public class TimedWaitTest {
+    public static void main(String[] args) throws InterruptedException {
+        CountDownLatch latch = new CountDownLatch(4);
+        for (int i = 0; i < 4; i++) {
+            int id = i;
+            WispEngine.dispatch(() -> {
+                final Object lock = new Object();
+                synchronized (lock) {
+                    try {
+                        lock.wait(1);
+                    } catch (InterruptedException e) {
+                        e.printStackTrace();
+                    }
+                }
+                latch.countDown();
+            });
+        }
+        latch.await();
+    }
+}
diff --git a/test/jdk/com/alibaba/wisp2/Wisp2ShutdownTest.java b/test/jdk/com/alibaba/wisp2/Wisp2ShutdownTest.java
new file mode 100644
index 00000000000..5eabd763686
--- /dev/null
+++ b/test/jdk/com/alibaba/wisp2/Wisp2ShutdownTest.java
@@ -0,0 +1,66 @@
+/*
+ * @test
+ * @summary Wisp2ShutdownTest
+ * @modules java.base/com.alibaba.wisp.engine:+open
+ * @library /lib/testlibrary
+ * @run main/othervm -XX:+EnableCoroutine -XX:+UseWispMonitor -Dcom.alibaba.wisp.transparentWispSwitch=true -Dcom.alibaba.wisp.version=2 Wisp2ShutdownTest
+ */
+
+import java.lang.reflect.Constructor;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.ThreadFactory;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import static jdk.testlibrary.Asserts.assertEQ;
+
+public class Wisp2ShutdownTest {
+    public static void main(String[] args) throws Exception {
+        AtomicInteger n = new AtomicInteger();
+
+        Constructor c = Class.forName("com.alibaba.wisp.engine.Wisp2Group")
+                .getDeclaredConstructor(int.class, ThreadFactory.class);
+        c.setAccessible(true);
+        ExecutorService g = (ExecutorService) c.newInstance(4, new ThreadFactory() {
+            @Override
+            public Thread newThread(Runnable r) {
+                return new Thread(r);
+            }
+        });
+
+        for (int i = 0; i < 888; i++) {
+            g.execute(() -> {
+                n.incrementAndGet();
+                try {
+                    while (true) {
+                        sleep(1000000);
+                    }
+                } finally {
+                    n.decrementAndGet();
+                }
+            });
+        }
+
+        while (n.get() != 888) {
+            n.get();
+        }
+
+        long start = System.currentTimeMillis();
+
+        g.shutdown();
+
+        g.awaitTermination(1, TimeUnit.SECONDS);
+
+        System.out.println(System.currentTimeMillis() - start + "ms");
+
+        assertEQ(n.get(), 0);
+    }
+
+    private static void sleep(long millis) {
+        try {
+            Thread.sleep(millis);
+        } catch (InterruptedException e) {
+            e.printStackTrace();
+        }
+    }
+}
diff --git a/test/jdk/com/alibaba/wisp2/Wisp2WaitNotifyTest.java b/test/jdk/com/alibaba/wisp2/Wisp2WaitNotifyTest.java
new file mode 100644
index 00000000000..ed1e85db07b
--- /dev/null
+++ b/test/jdk/com/alibaba/wisp2/Wisp2WaitNotifyTest.java
@@ -0,0 +1,70 @@
+/*
+ * @test
+ * @library /lib/testlibrary
+ * @summary Test Object.wait/notify with coroutine in wisp2
+ * @run main/othervm  -XX:-UseBiasedLocking -XX:+EnableCoroutine -XX:+UseWispMonitor -Dcom.alibaba.wisp.transparentWispSwitch=true -Dcom.alibaba.wisp.version=2 Wisp2WaitNotifyTest
+ */
+
+import com.alibaba.wisp.engine.WispEngine;
+
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import static jdk.testlibrary.Asserts.assertEQ;
+
+public class Wisp2WaitNotifyTest {
+
+    private AtomicInteger seq = new AtomicInteger();
+    private int finishCnt = 0;
+    private CountDownLatch latch = new CountDownLatch(1);
+    private boolean fooCond = false;
+
+    public static void main(String[] args) throws Exception {
+        Wisp2WaitNotifyTest s = new Wisp2WaitNotifyTest();
+        synchronized (s) {
+            WispEngine.dispatch(s::foo);
+            assertEQ(s.seq.getAndIncrement(), 0);
+        }
+
+        s.latch.await();
+
+        assertEQ(s.seq.getAndIncrement(), 5);
+        synchronized (s) {
+            while (s.finishCnt < 2) {
+                s.wait();
+            }
+        }
+        assertEQ(s.seq.getAndIncrement(), 6);
+    }
+
+    private synchronized void foo() {
+        assertEQ(seq.getAndIncrement(), 1);
+
+        WispEngine.dispatch(this::bar);
+        assertEQ(seq.getAndIncrement(), 2);
+        try {
+            while (!fooCond) {
+                wait();
+            }
+        } catch (InterruptedException e) {
+            e.printStackTrace();
+        }
+        assertEQ(seq.getAndIncrement(), 4);
+
+        latch.countDown();
+        finishCnt++;
+        notifyAll();
+    }
+
+    private void bar() {
+        synchronized (this) {
+            assertEQ(seq.getAndIncrement(), 3);
+            fooCond = true;
+            notifyAll();
+        }
+        synchronized (this) {
+            finishCnt++;
+            notifyAll();
+        }
+    }
+}
diff --git a/test/jdk/com/alibaba/wisp2/Wisp2WorkStealTest.java b/test/jdk/com/alibaba/wisp2/Wisp2WorkStealTest.java
new file mode 100644
index 00000000000..f9dc6938047
--- /dev/null
+++ b/test/jdk/com/alibaba/wisp2/Wisp2WorkStealTest.java
@@ -0,0 +1,63 @@
+/*
+ * @test
+ * @library /lib/testlibrary
+ * @summary verification of work stealing really happened
+ * @modules java.base/jdk.internal.misc
+ * @run main/othervm -XX:+UseWisp2 Wisp2WorkStealTest
+ */
+
+import com.alibaba.wisp.engine.WispEngine;
+import jdk.internal.access.SharedSecrets;
+
+import java.util.concurrent.atomic.AtomicReference;
+
+import static jdk.testlibrary.Asserts.assertNE;
+
+public class Wisp2WorkStealTest {
+    public static void main(String[] args) {
+        Object lock = new Object();
+        AtomicReference t = new AtomicReference<>();
+        AtomicReference t2 = new AtomicReference<>();
+        WispEngine.dispatch(() -> {
+            t.set(SharedSecrets.getJavaLangAccess().currentThread0());
+            synchronized (lock) {
+                try {
+                    lock.wait();
+                } catch (InterruptedException e) {
+                    e.printStackTrace();
+                }
+            }
+            t2.set(SharedSecrets.getJavaLangAccess().currentThread0());
+        });
+
+        // letting a coroutine occupy the carrier
+        while (true) {
+            AtomicReference found = new AtomicReference<>(null);
+            WispEngine.dispatch(() -> {
+                if (SharedSecrets.getJavaLangAccess().currentThread0() == t.get()) {
+                    found.set(true);
+                    long start = System.currentTimeMillis();
+                    while (System.currentTimeMillis() - start < 1000) {
+                        // occupy the carrier
+                    }
+                } else {
+                    found.set(false);
+                }
+            });
+            while (found.get() == null) {
+            }
+            if (found.get()) {
+                break;
+            }
+        }
+
+        synchronized (lock) {
+            lock.notify();
+        }
+
+        while (t2.get() == null) {
+        }
+
+        assertNE(t.get(), t2.get());
+    }
+}
diff --git a/test/jdk/com/alibaba/wisp2/Wisp2YieldTest.java b/test/jdk/com/alibaba/wisp2/Wisp2YieldTest.java
new file mode 100644
index 00000000000..c5ade47a334
--- /dev/null
+++ b/test/jdk/com/alibaba/wisp2/Wisp2YieldTest.java
@@ -0,0 +1,41 @@
+/*
+ * @test
+ * @library /lib/testlibrary
+ * @summary Test yield in wisp2
+ * @run main/othervm  -XX:-UseBiasedLocking -XX:+EnableCoroutine -XX:+UseWispMonitor -Dcom.alibaba.wisp.transparentWispSwitch=true -Dcom.alibaba.wisp.version=2 -Dcom.alibaba.wisp.workerEngines=1 Wisp2YieldTest
+ */
+
+import com.alibaba.wisp.engine.WispEngine;
+
+import static jdk.testlibrary.Asserts.assertTrue;
+
+
+public class Wisp2YieldTest {
+    public static void main(String[] args) {
+        boolean success[] = new boolean[1];
+        WispEngine.dispatch(() -> {
+            WispEngine.dispatch(() -> {
+                success[0] = true;
+            });
+            long start = System.currentTimeMillis();
+            while (System.currentTimeMillis() - start < 100) {
+            }
+            Thread.yield();
+            while (true) {
+
+            }
+        });
+
+        sleep(1000);
+
+        assertTrue(success[0]);
+    }
+
+    private static void sleep(long ms) {
+        try {
+            Thread.sleep(ms);
+        } catch (InterruptedException e) {
+            e.printStackTrace();
+        }
+    }
+}
diff --git a/test/jdk/com/alibaba/wisp2/bug/ConcurrentThreadJoinTest.java b/test/jdk/com/alibaba/wisp2/bug/ConcurrentThreadJoinTest.java
new file mode 100644
index 00000000000..cba11d95bbf
--- /dev/null
+++ b/test/jdk/com/alibaba/wisp2/bug/ConcurrentThreadJoinTest.java
@@ -0,0 +1,22 @@
+/*
+ * @test
+ * @library /lib/testlibrary
+ * @summary ensure thread.isAlive() is false after thread.join()
+ * @run main/othervm -XX:+UseWisp2 ConcurrentThreadJoinTest
+ */
+
+import static jdk.testlibrary.Asserts.assertFalse;
+
+public class ConcurrentThreadJoinTest {
+    public static void main(String[] args) throws Exception {
+        long start = System.currentTimeMillis();
+
+        while (System.currentTimeMillis() - start < 3000) {
+            Thread thread = new Thread(() -> {
+            });
+            thread.start();
+            thread.join(1000);
+            assertFalse(thread.isAlive());
+        }
+    }
+}
diff --git a/test/jdk/com/alibaba/wisp2/bug/DisableStealBugTest.java b/test/jdk/com/alibaba/wisp2/bug/DisableStealBugTest.java
new file mode 100644
index 00000000000..d5e4ecc6a84
--- /dev/null
+++ b/test/jdk/com/alibaba/wisp2/bug/DisableStealBugTest.java
@@ -0,0 +1,57 @@
+/*
+ * @test
+ * @library /lib/testlibrary
+ * @modules java.base/jdk.internal.misc
+ * @modules java.base/com.alibaba.wisp.engine:+open
+ * @summary test bug of update stealEnable fail
+ * @run main/othervm -XX:+UseWisp2 -XX:-EnableSteal DisableStealBugTest
+ */
+
+import com.alibaba.wisp.engine.WispEngine;
+import com.alibaba.wisp.engine.WispTask;
+import jdk.internal.access.SharedSecrets;
+
+import java.lang.reflect.Field;
+import java.util.concurrent.atomic.AtomicReference;
+
+import static jdk.testlibrary.Asserts.assertTrue;
+
+public class DisableStealBugTest {
+    public static void main(String[] args) throws Exception {
+        AtomicReference task = new AtomicReference<>();
+
+        WispEngine.dispatch(() -> {
+            task.set(SharedSecrets.getWispEngineAccess().getCurrentTask());
+            setOrGetStealEnable(task.get(), true, false);
+            try {
+                Thread.sleep(10);
+                Thread.sleep(1000);
+            } catch (InterruptedException e) {
+                e.printStackTrace();
+            }
+        });
+
+        Thread.sleep(2000);
+        boolean stealEnable = setOrGetStealEnable(task.get(), false, false);
+        assertTrue(stealEnable);
+    }
+
+    private static boolean setOrGetStealEnable(WispTask task, boolean isSet, boolean b) {
+        try {
+            Field resumeEntryField = task.getClass().getDeclaredField("resumeEntry");
+            resumeEntryField.setAccessible(true);
+            final Object resumeEntry = resumeEntryField.get(task);
+
+            Field stealEnableField = resumeEntry.getClass().getDeclaredField("stealEnable");
+            stealEnableField.setAccessible(true);
+            if (isSet) {
+                stealEnableField.setBoolean(resumeEntry, b);
+                return b;
+            } else {
+                return stealEnableField.getBoolean(resumeEntry);
+            }
+        } catch (ReflectiveOperationException e) {
+            throw new Error(e);
+        }
+    }
+}
diff --git a/test/jdk/com/alibaba/wisp2/bug/Wisp2ThreadObjLeakInThreadGroupTest.java b/test/jdk/com/alibaba/wisp2/bug/Wisp2ThreadObjLeakInThreadGroupTest.java
new file mode 100644
index 00000000000..43532427ddd
--- /dev/null
+++ b/test/jdk/com/alibaba/wisp2/bug/Wisp2ThreadObjLeakInThreadGroupTest.java
@@ -0,0 +1,32 @@
+/*
+ * @test
+ * @library /lib/testlibrary
+ * @summary test bug fix of thread object leak in thread group
+ * @run main/othervm -XX:+UseWisp2 Wisp2ThreadObjLeakInThreadGroupTest
+ */
+
+import java.util.concurrent.CountDownLatch;
+
+import static jdk.testlibrary.Asserts.assertEQ;
+
+public class Wisp2ThreadObjLeakInThreadGroupTest {
+    public static void main(String[] args) throws Exception {
+        ThreadGroup tg = Thread.currentThread().getThreadGroup();
+        int count = tg.activeCount();
+        CountDownLatch c1 = new CountDownLatch(1), c2 = new CountDownLatch(1);
+        Thread thread = new Thread(tg, () -> {
+            c1.countDown();
+            try {
+                c2.await();
+            } catch (InterruptedException e) {
+                e.printStackTrace();
+            }
+        }, Wisp2ThreadObjLeakInThreadGroupTest.class.getSimpleName());
+        thread.start();
+        c1.await(); // wait start
+        assertEQ(tg.activeCount(), count + 1);
+        c2.countDown(); // notify finish
+        thread.join();
+        assertEQ(tg.activeCount(), count);
+    }
+}
diff --git a/test/jdk/java/dyn/BasicStealTest.java b/test/jdk/java/dyn/BasicStealTest.java
new file mode 100644
index 00000000000..c878f3e7a75
--- /dev/null
+++ b/test/jdk/java/dyn/BasicStealTest.java
@@ -0,0 +1,57 @@
+/*
+ * @test
+ * @summary test basic coroutine steal mechanism
+ * @library /lib/testlibrary
+ * @run main/othervm -XX:+EnableCoroutine BasicStealTest
+ */
+
+import java.dyn.Coroutine;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.function.Consumer;
+
+import static jdk.testlibrary.Asserts.assertEQ;
+import static jdk.testlibrary.Asserts.assertTrue;
+
+public class BasicStealTest {
+    public static void main(String[] args) {
+        AtomicReference toBeStolen = new AtomicReference<>();
+        Thread main = Thread.currentThread();
+
+        Thread t = new Thread(() -> {
+            Coroutine threadCoro = Thread.currentThread().getCoroutineSupport().threadCoroutine();
+
+            Coroutine coro = new Coroutine(() -> {
+                AtomicReference> foo = new AtomicReference<>();
+                foo.set((x) -> {
+                    if (x == 10) {
+                        Coroutine.yieldTo(threadCoro);
+                    } else {
+                        System.out.println(x + " enter " + Thread.currentThread());
+                        foo.get().accept(x + 1);
+                        System.out.println(x + " exit" + Thread.currentThread());
+                        assertEQ(Thread.currentThread(), main);
+                    }
+                });
+                foo.get().accept(0);
+            });
+
+            Coroutine.yieldTo(coro);
+            // switch from foo()...
+            toBeStolen.set(coro);
+            try {
+                Thread.sleep(100000);
+            } catch (InterruptedException e) {
+                e.printStackTrace();
+            }
+        }, "new_thread");
+        t.setDaemon(true);
+        t.start();
+
+        while (toBeStolen.get() == null) {
+        }
+
+        assertTrue(toBeStolen.get().steal(false));
+        Coroutine.yieldTo(toBeStolen.get());
+
+    }
+}
diff --git a/test/jdk/java/dyn/ConcurrentStealTest.java b/test/jdk/java/dyn/ConcurrentStealTest.java
new file mode 100644
index 00000000000..9513114d62a
--- /dev/null
+++ b/test/jdk/java/dyn/ConcurrentStealTest.java
@@ -0,0 +1,106 @@
+/* @test
+ * @summary unit tests for steal in concurrent situation
+ * @run junit/othervm/timeout=300 -XX:+EnableCoroutine ConcurrentStealTest
+ */
+
+import org.junit.Test;
+
+import java.dyn.Coroutine;
+import java.util.Arrays;
+import java.util.Random;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ThreadLocalRandom;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import static org.junit.Assert.*;
+
+public class ConcurrentStealTest {
+
+    @Test
+    public void stealRunning() throws Exception {
+        Coroutine[] coro = new Coroutine[1];
+
+        CountDownLatch cdl = new CountDownLatch(1);
+
+        Thread t = new Thread(() -> {
+            coro[0] = new Coroutine(() -> {
+                try {
+                    cdl.countDown();
+                    Thread.sleep(10000000);
+                } catch (InterruptedException e) {
+                    e.printStackTrace();
+                }
+            });
+            Coroutine.yieldTo(coro[0]);
+        }, "stealRunning");
+        t.setDaemon(true);
+        t.start();
+
+        cdl.await();
+
+        Thread.sleep(500);
+
+        assertFalse(coro[0].steal(true));
+    }
+
+    final static int THREADS = Math.min(Runtime.getRuntime().availableProcessors(), 8);
+    final static int CORO_PER_TH = 20;
+    final static int TIMEOUT = 10;
+
+    @Test
+    public void randomSteal() throws Exception {
+        Coroutine[] coro = new Coroutine[THREADS * CORO_PER_TH];
+        int[] cnt = new int[THREADS];
+
+        AtomicInteger sync = new AtomicInteger();
+
+        for (int th = 1; th < THREADS; th++) {
+            int cth = th;
+            Thread t = new Thread(() -> {
+                for (int i = 0; i < CORO_PER_TH; i++) {
+                    coro[CORO_PER_TH * cth + i] = new Coroutine(() -> {
+                        while (true) {
+                            cnt[cth]++;
+                            Coroutine.yield();
+                        }
+                    });
+                }
+                sync.incrementAndGet();
+                while (sync.get() != THREADS) {
+                }
+
+                runRandomCoroutines(System.nanoTime(), coro);
+            }, "randomSteal-" + th);
+            t.setDaemon(true);
+            t.start();
+        }
+
+        for (int i = 0; i < CORO_PER_TH; i++) {
+            coro[i] = new Coroutine(() -> {
+                while (true) {
+                    cnt[0]++;
+                    Coroutine.yield();
+                }
+            });
+        }
+
+        sync.incrementAndGet();
+        while (sync.get() != THREADS) {
+        }
+        runRandomCoroutines(System.nanoTime(), coro);
+        for (int i = 0; i < cnt.length; i++) {
+            cnt[i] /= TIMEOUT;
+        }
+        System.out.println(Arrays.toString(cnt));
+    }
+
+    private static void runRandomCoroutines(long start, Coroutine[] coro) {
+        while (System.nanoTime() - start < TimeUnit.SECONDS.toNanos(TIMEOUT)) {
+            Coroutine co = coro[ThreadLocalRandom.current().nextInt(coro.length)];
+            if (co.getThread() == Thread.currentThread() || co.steal(true)) {
+                Coroutine.yieldTo(co);
+            }
+        }
+    }
+}

From 729937cc6565d6976037ee4f758281ad944cf145 Mon Sep 17 00:00:00 2001
From: Xiaomin 
Date: Thu, 29 Jun 2023 11:40:45 +0800
Subject: [PATCH 2/2] [Misc] Fix bugs in previous commit.

Summary:
- Basic change such as naming convention or typo.
- Fix wrong usage of some functions.
- Add one more dereference when encouter _threadObj since it's oop* rather than oop in Java11.
- Add WispThread convertion in clinit_barrier when UseWispMonitor is enabled, because ini thread is WispThread.
- Pass correct current thread to ExceptionMark constructor in Runtime1::monitorexit_wisp.
- Modify code in test to pass hotspot and jdk jtregs.
- Fix wisp2 create property crash.
- Delete control node in make_runtime_call() since it is wrong that treating control node to memory node.
- Refine coroutine properties append logic(From JVM_GetProperties to Arguments::init_system_properties).
- Hoist code of safepoint state out of MutexLocker to prevent safepoint deadlock problem.
- Fix WispThread BarrierSet and JavaCalls assert.
- Fix Thread interrupted flag in public class Thread.
- Fix threadObj changed in coroutine steal.
- Add more debug functions for switchTo.
- Delete jniDetachThreadHoldingMonitorTest.sh, jniMonitorExitTest.sh and SelectorInitCriticalTest.java because SelectorProvider removed lock in Java17.
- test/hotspot/jtreg/runtime/jni/registerNativesWarning/TestRegisterNativesWarning.java: yield() in Thread.java is not a native method after we port wisp coroutine to JDK. Thus we use currentTimeMillis() instead.
- vmTestbase/nsk/monitoring/stress/thread/strace001/TestDescription.java: Add "java.lang.Thread.sleep0" and "java.lang.Thread.yield0" in expectedTrace, and let expectedLength = depth + 4.
- test/hotspot/jtreg/vmTestbase/nsk/monitoring/share/ThreadController.java: Add sleep0 and yield0 in expectedMethods.
- test/hotspot/jtreg/vmTestbase/nsk/stress/strace/strace00*.java: Add sleep0 and yield0 in EXPECTED_METHODS.
- test/hotspot/jtreg/vmTestbase/nsk/monitoring/share/thread/SleepingThread.java and RunningThread.java: Add sleep0 and yield0 in expectedMethods.
- test/hotspot/jtreg/vmTestbase/nsk/share/locks/LockingThread.java: Add 1 to expectedDepth and add function myWait to add 1 more stack frame.
- test/jdk/java/beans/XMLDecoder/8028054/Task.java: Remove "java.dyn.CoroutineSupport" from fileNames.
- test/jdk/jdk/jfr/event/runtime/TestSystemPropertyEvent.java: Add "com.alibaba.coroutine.enableCoroutine" to SystemProperty.
- test/jdk/com/alibaba/wisp2/Wisp2ShutdownTest.java: Change awaitTermination time to 5 seconds, because all coroutines may not terminated in 1 second.
- Put trace_switch, print_site and coroutine_switch_trace into debug version.

Reviewed-by: yulei

Test Plan:
test/hotspot/jtreg/runtime/coroutine/
test/jdk/com/alibaba/wisp
test/jdk/com/alibaba/wisp2

Issue:
https://github.com/dragonwell-project/dragonwell17/issues/77
---
 make/modules/java.base/Copy.gmk               | 16 ++--
 src/hotspot/cpu/x86/c2_MacroAssembler_x86.cpp |  7 ++
 src/hotspot/cpu/x86/interp_masm_x86.cpp       |  6 +-
 src/hotspot/cpu/x86/macroAssembler_x86.cpp    |  7 ++
 src/hotspot/cpu/x86/sharedRuntime_x86_64.cpp  | 47 +++++++++---
 src/hotspot/os/posix/os_posix.cpp             | 14 ----
 src/hotspot/share/c1/c1_Runtime1.cpp          |  8 +-
 src/hotspot/share/classfile/javaClasses.cpp   | 11 ++-
 .../share/classfile/systemDictionary.cpp      | 32 ++++----
 .../share/classfile/systemDictionary.hpp      |  6 +-
 src/hotspot/share/gc/shared/barrierSet.cpp    |  4 +
 .../share/interpreter/interpreterRuntime.cpp  | 10 ++-
 .../share/interpreter/interpreterRuntime.hpp  |  5 ++
 src/hotspot/share/jvmci/jvmciRuntime.cpp      |  2 +-
 src/hotspot/share/oops/weakHandle.hpp         |  2 +-
 src/hotspot/share/opto/graphKit.cpp           | 26 +++----
 src/hotspot/share/opto/graphKit.hpp           |  3 +-
 src/hotspot/share/opto/library_call.cpp       | 30 +++++++-
 src/hotspot/share/prims/jvm.cpp               | 11 +--
 .../share/prims/universalUpcallHandler.cpp    |  4 +-
 src/hotspot/share/runtime/coroutine.cpp       | 31 +++++---
 src/hotspot/share/runtime/coroutine.hpp       |  4 +-
 src/hotspot/share/runtime/handles.hpp         |  1 -
 src/hotspot/share/runtime/javaCalls.cpp       |  7 +-
 src/hotspot/share/runtime/mutexLocker.cpp     |  8 +-
 src/hotspot/share/runtime/mutexLocker.hpp     | 21 ++++--
 src/hotspot/share/runtime/objectMonitor.cpp   | 54 +++++++++-----
 src/hotspot/share/runtime/os.hpp              |  2 -
 src/hotspot/share/runtime/reflection.cpp      |  9 +--
 src/hotspot/share/runtime/sharedRuntime.cpp   |  9 ++-
 src/hotspot/share/runtime/sharedRuntime.hpp   |  5 ++
 src/hotspot/share/runtime/synchronizer.cpp    | 74 +++++++++++++------
 src/hotspot/share/runtime/synchronizer.hpp    | 42 +++++++----
 src/hotspot/share/runtime/thread.cpp          | 28 +++++--
 src/hotspot/share/runtime/thread.hpp          |  4 +-
 src/hotspot/share/runtime/vframe.hpp          |  1 +
 src/hotspot/share/utilities/exceptions.cpp    |  5 --
 .../share/classes/java/dyn/CoroutineBase.java |  2 +-
 .../share/classes/java/lang/Thread.java       |  4 +-
 .../classes/java/net/DatagramSocket.java      | 20 +++--
 .../sun/nio/ch/ServerSocketChannelImpl.java   |  6 +-
 .../coroutine/C1ThrowSyncExceptionTest.java   |  2 +-
 .../TestAvoidDeoptCoroutineMethod.java        |  2 +-
 .../runtime/coroutine/TimeSliceSyncTest.java  |  2 +-
 .../runtime/coroutine/Wisp2SwitchTest.java    |  2 +-
 .../runtime/coroutine/Wisp2SwitchTest2.java   |  2 +-
 .../jniDetachThreadHoldingMonitorTest.sh      | 18 -----
 .../runtime/coroutine/jniMonitorExitTest.sh   | 19 -----
 .../TestRegisterNativesWarning.java           | 11 ++-
 .../libregisterNativesWarning.c               |  7 +-
 .../monitoring/share/ThreadController.java    |  2 +
 .../share/thread/RunningThread.java           |  3 +-
 .../share/thread/SleepingThread.java          |  1 +
 .../monitoring/stress/thread/strace001.java   | 14 +++-
 .../nsk/share/locks/LockingThread.java        | 17 ++++-
 .../nsk/stress/strace/strace001.java          |  1 +
 .../nsk/stress/strace/strace002.java          |  1 +
 .../nsk/stress/strace/strace003.java          |  1 +
 .../nsk/stress/strace/strace004.java          |  1 +
 .../nsk/stress/strace/strace005.java          |  1 +
 .../nsk/stress/strace/strace006.java          |  1 +
 .../nsk/stress/strace/strace007.java          |  1 +
 .../nsk/stress/strace/strace008.java          |  1 +
 .../nsk/stress/strace/strace009.java          |  1 +
 .../nsk/stress/strace/strace010.java          |  1 +
 .../nsk/stress/strace/strace011.java          |  1 +
 .../nsk/stress/strace/strace012.java          |  1 +
 test/jdk/com/alibaba/wisp/ExecutionTest.java  |  2 +-
 test/jdk/com/alibaba/wisp/IoTest.java         |  2 +-
 test/jdk/com/alibaba/wisp/ParkTest.java       |  2 +-
 .../alibaba/wisp/SelectorLazyCreateTest.java  |  6 +-
 .../wisp/boot/UnsafeDependencyBugTest.java    |  4 +-
 .../wisp/bug/CancelTimerAndSleepTest.java     |  2 +-
 .../com/alibaba/wisp/bug/ClearEventTest.java  |  2 +-
 .../alibaba/wisp/bug/Id2TaskMapLeakTest.java  |  4 +-
 .../bug/ReleaseWispSocketAndExitTest.java     |  2 +-
 .../wisp/bug/ResetTaskCancelTimerBugTest.java |  4 +-
 .../wisp/bug/SelectorInitCriticalTest.java    | 32 --------
 .../alibaba/wisp/bug/TestThreadStackTrace.sh  |  6 +-
 .../com/alibaba/wisp/bug/ThreadLockTest.java  |  4 +-
 .../bug/WispEngineCriticalSectionTest.java    |  8 +-
 .../WispSocketLeakWhenConnectTimeoutTest.java |  4 +-
 .../wisp/env/CtxClassLoaderIsolateTest.java   |  4 +-
 .../alibaba/wisp/io/CreateFdOnDemandTest.java |  6 +-
 .../alibaba/wisp/io/DatagramSocketTest.java   |  6 +-
 .../com/alibaba/wisp/io/GlobalPollerTest.java |  6 +-
 .../wisp/io/ReuseUdpSocektBufTest.java        |  4 +-
 test/jdk/com/alibaba/wisp/lock/AQSTest.java   |  2 +-
 .../alibaba/wisp/lock/ElisionSpinTest.java    |  8 +-
 test/jdk/com/alibaba/wisp/lock/LockTest.java  |  2 +-
 .../wisp/lock/LockUninterruptiblyTest.java    |  4 +-
 .../com/alibaba/wisp/lock/UnsafeParkTest.java |  4 +-
 .../alibaba/wisp/monitor/JNICriticalTest.java |  2 +-
 .../wisp/monitor/LazyUnparkBugTest.java       |  2 +-
 .../alibaba/wisp/monitor/MultiThreadTest.java |  4 +-
 .../alibaba/wisp/monitor/WaitNotifyTest.java  |  4 +-
 .../DisableThreadAsWispAtRuntimeTest.java     |  6 +-
 .../wisp/thread/EngineExecutorTest.java       |  2 +-
 .../wisp/thread/InterruptedSleepTest.java     |  8 +-
 .../com/alibaba/wisp/thread/IsAliveTest.java  |  6 +-
 .../wisp/thread/SubmittedTaskLimitTest.java   |  8 +-
 .../alibaba/wisp/thread/ThreadAsWispTest.java |  8 +-
 .../alibaba/wisp/thread/ThrowErrorTest.java   |  4 +-
 .../alibaba/wisp/thread/TimeSliceTest.java    |  4 +-
 .../com/alibaba/wisp/thread/YieldTest.java    |  4 +-
 .../wisp/timer/DaemonThreadGroupTest.java     |  6 +-
 .../com/alibaba/wisp/timer/OverflowTest.java  |  6 +-
 .../wisp/timer/PriorityQueueSortTest.java     |  4 +-
 .../com/alibaba/wisp/timer/SleepRPCTest.java  |  4 +-
 .../jdk/com/alibaba/wisp/timer/SleepTest.java |  4 +-
 .../jdk/com/alibaba/wisp/timer/TimerTest.java |  2 +-
 .../alibaba/wisp2/AllThreadAsWispTest.java    |  6 +-
 .../wisp2/CtxClassLoaderInheritanceTest.java  |  4 +-
 test/jdk/com/alibaba/wisp2/DispatchTest.java  |  2 +-
 test/jdk/com/alibaba/wisp2/HandOffTest.java   |  4 +-
 .../alibaba/wisp2/NioBlockingAcceptTest.java  |  4 +-
 .../ReuseWispTaskAfterThreadJoinTest.java     |  2 +-
 .../jdk/com/alibaba/wisp2/ThreadJoinTest.java |  2 +-
 .../com/alibaba/wisp2/Wisp2ShutdownTest.java  |  7 +-
 .../alibaba/wisp2/Wisp2WaitNotifyTest.java    |  4 +-
 .../com/alibaba/wisp2/Wisp2WorkStealTest.java |  6 +-
 .../jdk/com/alibaba/wisp2/Wisp2YieldTest.java |  4 +-
 .../wisp2/bug/ConcurrentThreadJoinTest.java   |  4 +-
 .../wisp2/bug/DisableStealBugTest.java        |  6 +-
 .../Wisp2ThreadObjLeakInThreadGroupTest.java  |  4 +-
 .../java/beans/XMLDecoder/8028054/Task.java   |  5 ++
 test/jdk/java/dyn/BasicStealTest.java         |  6 +-
 .../runtime/TestSystemPropertyEvent.java      |  5 ++
 128 files changed, 563 insertions(+), 432 deletions(-)
 delete mode 100644 test/hotspot/jtreg/runtime/coroutine/jniDetachThreadHoldingMonitorTest.sh
 delete mode 100644 test/hotspot/jtreg/runtime/coroutine/jniMonitorExitTest.sh
 delete mode 100644 test/jdk/com/alibaba/wisp/bug/SelectorInitCriticalTest.java

diff --git a/make/modules/java.base/Copy.gmk b/make/modules/java.base/Copy.gmk
index b2ceb409d6a..b581017281d 100644
--- a/make/modules/java.base/Copy.gmk
+++ b/make/modules/java.base/Copy.gmk
@@ -196,6 +196,15 @@ $(NET_PROPERTIES_DST): $(NET_PROPERTIES_SRCS)
 
 TARGETS += $(NET_PROPERTIES_DST)
 
+################################################################################
+
+$(eval $(call SetupCopyFiles, COPY_WISP_PROPERTIES, \
+    FILES := $(TOPDIR)/src/java.base/share/conf/wisp.properties, \
+    DEST := $(CONF_DST_DIR), \
+))
+
+TARGETS += $(COPY_WISP_PROPERTIES)
+
 ifeq ($(call isTargetOs, linux), true)
   $(eval $(call SetupCopyFiles, COPY_SDP_CONF, \
       FILES := $(TOPDIR)/src/java.base/$(OPENJDK_TARGET_OS_TYPE)/conf/sdp/sdp.conf.template, \
@@ -205,13 +214,6 @@ ifeq ($(call isTargetOs, linux), true)
   TARGETS += $(COPY_SDP_CONF)
 endif
 
-$(eval $(call SetupCopyFiles, COPY_WISP_PROPERTIES, \
-    FILES := $(TOPDIR)/src/java.base/share/conf/wisp.properties, \
-    DEST := $(CONF_DST_DIR), \
-))
-
-TARGETS += $(COPY_WISP_PROPERTIES)
-
 ################################################################################
 
 # JDK license and assembly exception files to be packaged in JMOD
diff --git a/src/hotspot/cpu/x86/c2_MacroAssembler_x86.cpp b/src/hotspot/cpu/x86/c2_MacroAssembler_x86.cpp
index 9bc73c386d2..26feaf6a88e 100644
--- a/src/hotspot/cpu/x86/c2_MacroAssembler_x86.cpp
+++ b/src/hotspot/cpu/x86/c2_MacroAssembler_x86.cpp
@@ -615,7 +615,14 @@ void C2_MacroAssembler::fast_lock(Register objReg, Register boxReg, Register tmp
   // Propagate ICC.ZF from CAS above into DONE_LABEL.
   jcc(Assembler::equal, DONE_LABEL);           // CAS above succeeded; propagate ZF = 1 (success)
 
+  if (UseWispMonitor) {
+    movptr (r15_thread, Address(r15_thread, JavaThread::current_coroutine_offset()));
+    movptr (r15_thread, Address(r15_thread, Coroutine::wisp_thread_offset()));
+  }
   cmpptr(r15_thread, rax);                     // Check if we are already the owner (recursive lock)
+  if (UseWispMonitor) {
+    movptr (r15_thread, Address(r15_thread, WispThread::thread_offset()));
+  }
   jcc(Assembler::notEqual, DONE_LABEL);        // If not recursive, ZF = 0 at this point (fail)
   incq(Address(scrReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(recursions)));
   xorq(rax, rax); // Set ZF = 1 (success) for recursive lock, denoting locking success
diff --git a/src/hotspot/cpu/x86/interp_masm_x86.cpp b/src/hotspot/cpu/x86/interp_masm_x86.cpp
index 16550f78bd4..77f02d94bc9 100644
--- a/src/hotspot/cpu/x86/interp_masm_x86.cpp
+++ b/src/hotspot/cpu/x86/interp_masm_x86.cpp
@@ -817,11 +817,11 @@ void InterpreterMacroAssembler::jump_from_interpreted(Register method, Register
     cmpb(Address(temp, JavaThread::interp_only_mode_offset()), 0);
     jccb(Assembler::zero, run_compiled_code);
     if (EnableCoroutine) {
-      cmpq(Address(method, Method::intrinsic_id_offset_in_bytes()), (int)vmIntrinsics::_switchTo);
+      cmpq(Address(method, Method::intrinsic_id_offset_in_bytes()), (int32_t)vmIntrinsics::_switchTo);
       jcc(Assembler::zero, coroutine_skip_interpret);
-      cmpq(Address(method, Method::intrinsic_id_offset_in_bytes()), (int)vmIntrinsics::_switchToAndExit);
+      cmpq(Address(method, Method::intrinsic_id_offset_in_bytes()), (int32_t)vmIntrinsics::_switchToAndExit);
       jcc(Assembler::zero, coroutine_skip_interpret);
-      cmpq(Address(method, Method::intrinsic_id_offset_in_bytes()), (int)vmIntrinsics::_switchToAndTerminate);
+      cmpq(Address(method, Method::intrinsic_id_offset_in_bytes()), (int32_t)vmIntrinsics::_switchToAndTerminate);
       jcc(Assembler::zero, coroutine_skip_interpret);
     }
     jmp(Address(method, Method::interpreter_entry_offset()));
diff --git a/src/hotspot/cpu/x86/macroAssembler_x86.cpp b/src/hotspot/cpu/x86/macroAssembler_x86.cpp
index 7fb116aec13..ded490b354a 100644
--- a/src/hotspot/cpu/x86/macroAssembler_x86.cpp
+++ b/src/hotspot/cpu/x86/macroAssembler_x86.cpp
@@ -4161,8 +4161,15 @@ void MacroAssembler::clinit_barrier(Register klass, Register thread, Label* L_fa
   cmpb(Address(klass, InstanceKlass::init_state_offset()), InstanceKlass::fully_initialized);
   jcc(Assembler::equal, *L_fast_path);
 
+  if (UseWispMonitor) {
+    movptr(thread, Address(thread, JavaThread::current_coroutine_offset()));
+    movptr(thread, Address(thread, Coroutine::wisp_thread_offset()));
+  }
   // Fast path check: current thread is initializer thread
   cmpptr(thread, Address(klass, InstanceKlass::init_thread_offset()));
+  if (UseWispMonitor) {
+    movptr(thread, Address(thread, WispThread::thread_offset()));
+  }
   if (L_slow_path == &L_fallthrough) {
     jcc(Assembler::equal, *L_fast_path);
     bind(*L_slow_path);
diff --git a/src/hotspot/cpu/x86/sharedRuntime_x86_64.cpp b/src/hotspot/cpu/x86/sharedRuntime_x86_64.cpp
index 649a4dec9cf..b9e920f5eee 100644
--- a/src/hotspot/cpu/x86/sharedRuntime_x86_64.cpp
+++ b/src/hotspot/cpu/x86/sharedRuntime_x86_64.cpp
@@ -1531,7 +1531,11 @@ static void gen_special_dispatch(MacroAssembler* masm,
                                                  receiver_reg, member_reg, /*for_compiler_entry:*/ true);
 }
 
-void create_switchTo_contents(MacroAssembler *masm, int start, OopMapSet* oop_maps, int &stack_slots, int total_in_args, BasicType *in_sig_bt, VMRegPair *in_regs, BasicType ret_type, bool terminate);
+void create_switchTo_contents(MacroAssembler *masm, int start,
+  OopMapSet* oop_maps, int &stack_slots,
+  int total_in_args, BasicType *in_sig_bt, VMRegPair *in_regs,
+  BasicType ret_type, bool terminate,
+  int total_c_args, VMRegPair *out_regs);
 
 void generate_thread_fix(MacroAssembler *masm, Method *method) {
   // we can have a check here at the codegen time, so no cost in runtime.
@@ -1823,10 +1827,14 @@ nmethod* SharedRuntime::generate_native_wrapper(MacroAssembler* masm,
   if (EnableCoroutine) {
     // the coroutine support methods have a hand-coded fast version that will handle the most common cases
     if (method->intrinsic_id() == vmIntrinsics::_switchTo) {
-      create_switchTo_contents(masm, start, oop_maps, stack_slots, total_in_args, in_sig_bt, in_regs, ret_type, false);
+      create_switchTo_contents(masm, start, oop_maps, stack_slots,
+        total_in_args, in_sig_bt, in_regs, ret_type, false,
+        total_c_args, out_regs);
     } else if (method->intrinsic_id() == vmIntrinsics::_switchToAndTerminate ||
         method->intrinsic_id() == vmIntrinsics::_switchToAndExit) {
-      create_switchTo_contents(masm, start, oop_maps, stack_slots, total_in_args, in_sig_bt, in_regs, ret_type, true);
+      create_switchTo_contents(masm, start, oop_maps, stack_slots,
+        total_in_args, in_sig_bt, in_regs, ret_type, true,
+        total_c_args, out_regs);
     }
   }
 
@@ -3991,8 +3999,26 @@ MacroAssembler* debug_line(MacroAssembler* masm, int l) {
   return masm;
 }
 
+static const int64_t invalid_val = 0x500000005L;
+void crash_if_reg_invalid(MacroAssembler *masm, Register reg, int loc) {
+  Label skip;
+  __ cmp64(reg, ExternalAddress((address)&invalid_val));
+  __ jcc(Assembler::notEqual, skip);
+  __ movptr(Address(reg, (ByteSize)loc), (intptr_t)loc);
+  __ bind(skip);
+}
+
+#ifdef ASSERT
+void trace_switch(MacroAssembler *masm, Register reg, int total_c_args, VMRegPair *out_regs) {
+  save_args(masm, total_c_args, 0, out_regs);
+  __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::coroutine_switch_trace), r15_thread, reg);
+  restore_args(masm, total_c_args, 0, out_regs);
+}
+#endif
+
 void create_switchTo_contents(MacroAssembler *masm, int start, OopMapSet* oop_maps, int &stack_slots, int total_in_args,
-                              BasicType *in_sig_bt, VMRegPair *in_regs, BasicType ret_type, bool terminate) {
+                              BasicType *in_sig_bt, VMRegPair *in_regs, BasicType ret_type, bool terminate,
+                              int total_c_args, VMRegPair *out_regs) {
   assert(total_in_args == 2, "wrong number of arguments");
 
   if (j_rarg0 != rsi) {
@@ -4033,7 +4059,6 @@ void create_switchTo_contents(MacroAssembler *masm, int start, OopMapSet* oop_ma
     DEBUG_ONLY(stop_if_null(masm, old_coroutine_obj, "null old_coroutine"));
     __ movptr(old_coroutine, Address(old_coroutine_obj, java_dyn_CoroutineBase::get_data_offset()));
     DEBUG_ONLY(stop_if_null(masm, old_coroutine, "old_coroutine without data"));
-    __ movptr(old_stack, Address(old_coroutine, Coroutine::stack_offset()));
 
 #if defined(_WINDOWS)
     // rescue the SEH pointer
@@ -4059,14 +4084,18 @@ void create_switchTo_contents(MacroAssembler *masm, int start, OopMapSet* oop_ma
     __ movptr(Address(old_coroutine, Coroutine::last_Java_pc_offset()), temp);
     __ movptr(temp, Address(thread, JavaThread::last_Java_sp_offset()));
     __ movptr(Address(old_coroutine, Coroutine::last_Java_sp_offset()), temp);
-    // __ movptr(temp, Address(thread, JavaThread::privileged_stack_top_offset()));
-    // __ movptr(Address(old_coroutine, Coroutine::privileged_stack_top_offset()), temp);
     __ movptr(temp, Address(thread, JavaThread::threadObj_offset()));
+    // Now temp is JavaThread::_threadObj.
+    // In Java17, JavaThread::_threadObj is oop* while Java11 stores JavaThread::_threadObj as oop.
+    // So we dereference temp here to get the target JavaThread oop.
+    __ movptr(temp, Address(temp, 0));
     __ movl(temp, Address(temp, java_lang_Thread::thread_status_offset()));
     __ movl(Address(old_coroutine, Coroutine::thread_status_offset()), temp);
     __ movl(temp, Address(thread, JavaThread::java_call_counter_offset()));
     __ movl(Address(old_coroutine, Coroutine::java_call_counter_offset()), temp);
 
+    // store rsp into CorotineStack
+    __ movptr(old_stack, Address(old_coroutine, Coroutine::stack_offset()));
     __ movptr(Address(old_stack, CoroutineStack::last_sp_offset()), rsp);
   }
   Register target_stack = r12;
@@ -4100,10 +4129,10 @@ void create_switchTo_contents(MacroAssembler *masm, int start, OopMapSet* oop_ma
       __ movptr(Address(thread, JavaThread::last_Java_pc_offset()), temp);
       __ movptr(temp, Address(target_coroutine, Coroutine::last_Java_sp_offset()));
       __ movptr(Address(thread, JavaThread::last_Java_sp_offset()), temp);
-      // __ movptr(temp, Address(target_coroutine, Coroutine::privileged_stack_top_offset()));
-      // __ movptr(Address(thread, JavaThread::privileged_stack_top_offset()), temp);
       __ movl(temp2, Address(target_coroutine, Coroutine::thread_status_offset()));
       __ movptr(temp, Address(thread, JavaThread::threadObj_offset()));
+      // Dereference temp here to get the target JavaThread oop.
+      __ movptr(temp, Address(temp, 0));
       __ movl(Address(temp, java_lang_Thread::thread_status_offset()), temp2);
       __ movl(temp, Address(target_coroutine, Coroutine::java_call_counter_offset()));
       __ movl(Address(thread, JavaThread::java_call_counter_offset()), temp);
diff --git a/src/hotspot/os/posix/os_posix.cpp b/src/hotspot/os/posix/os_posix.cpp
index 8bb484c4ac8..5c34a30a07d 100644
--- a/src/hotspot/os/posix/os_posix.cpp
+++ b/src/hotspot/os/posix/os_posix.cpp
@@ -24,7 +24,6 @@
 
 
 #include "jvm.h"
-#include "memory/allocation.hpp"
 #ifdef LINUX
 #include "classfile/classLoader.hpp"
 #endif
@@ -876,19 +875,6 @@ void os::naked_short_sleep(jlong ms) {
   return;
 }
 
-bool clear_interrupt_for_wisp(Thread* thread) {
-  assert(EnableCoroutine, "Coroutine is disabled");
-  // If we only use -XX:+EnableCoroutine and -Dcom.alibaba.transparentAsync=true, we will
-  // fall here, so we cannot use `assert(UseWispMonitor)` only.
-  if (UseWispMonitor && thread->is_Wisp_thread()) {
-    thread = ((WispThread *)thread)->thread();
-  }
-  bool interrupted = ((JavaThread*) thread->as_Java_thread())->is_interrupted(false);
-  thread->osthread()->set_interrupted(false);
-
-  return interrupted;
-}
-
 char* os::Posix::describe_pthread_attr(char* buf, size_t buflen, const pthread_attr_t* attr) {
   size_t stack_size = 0;
   size_t guard_size = 0;
diff --git a/src/hotspot/share/c1/c1_Runtime1.cpp b/src/hotspot/share/c1/c1_Runtime1.cpp
index 351204ac829..77f16cc93e5 100644
--- a/src/hotspot/share/c1/c1_Runtime1.cpp
+++ b/src/hotspot/share/c1/c1_Runtime1.cpp
@@ -716,8 +716,8 @@ JRT_END
 JRT_ENTRY_NO_ASYNC(void, Runtime1::monitorexit_wisp(JavaThread* current, BasicObjectLock* lock))
   NOT_PRODUCT(_monitorexit_slowcase_cnt++;)
   assert(UseWispMonitor, "UseWispMonitor is off");
-  JavaThread* thread_tmp = NULL;
-  ExceptionMark __em(thread_tmp);
+  assert(current == Thread::current(), "sanity check");
+  ExceptionMark em(current);
   oop obj = lock->obj();
   // Almost a copy from Runtime1::monitorexit,
   // excpet that handles are used to access objects.
@@ -733,7 +733,7 @@ JRT_END
 // 3. There is no exception handler in this method, So it needs to unwind to its caller
 // 4. GC happened during unpark
 // This path will not call Java, so JRT_LEAF is used.
-JRT_LEAF(void, Runtime1::monitorexit_wisp_proxy(JavaThread* thread, BasicObjectLock* lock))
+JRT_LEAF(void, Runtime1::monitorexit_wisp_proxy(JavaThread* current, BasicObjectLock* lock))
   NOT_PRODUCT(_monitorexit_slowcase_cnt++;)
   assert(UseWispMonitor, "UseWispMonitor is off");
   EXCEPTION_MARK;
@@ -741,7 +741,7 @@ JRT_LEAF(void, Runtime1::monitorexit_wisp_proxy(JavaThread* thread, BasicObjectL
   assert(oopDesc::is_oop(obj), "must be NULL or an object");
   // Setting _is_proxy_unpark of current wisp thread to true.
   // Proxy unpark will be used when this flag is true.
-  WispThread* wisp_thread = WispThread::current(thread);
+  WispThread* wisp_thread = WispThread::current(current);
   wisp_thread->set_proxy_unpark_flag();
   // When using fast locking, the compiled code has already tried the fast case
   ObjectSynchronizer::exit(obj, lock->lock(), THREAD);
diff --git a/src/hotspot/share/classfile/javaClasses.cpp b/src/hotspot/share/classfile/javaClasses.cpp
index e0802196c1a..ef995e01fc5 100644
--- a/src/hotspot/share/classfile/javaClasses.cpp
+++ b/src/hotspot/share/classfile/javaClasses.cpp
@@ -4755,9 +4755,9 @@ void java_dyn_CoroutineBase::set_data(oop obj, jlong value) {
 int com_alibaba_wisp_engine_WispEngine::_isInCritical_offset = 0;
 
 void com_alibaba_wisp_engine_WispEngine::compute_offsets() {
-  Klass* k = vmClasses::com_alibaba_wisp_engine_WispEngine_klass();
-  assert(k != NULL, "WispEngine_klass is null");
-  compute_offset(_isInCritical_offset, InstanceKlass::cast(k), vmSymbols::isInCritical_name(),         vmSymbols::bool_signature());
+  InstanceKlass* ik = vmClasses::com_alibaba_wisp_engine_WispEngine_klass();
+  assert(ik != NULL, "WispEngine_klass is null");
+  compute_offset(_isInCritical_offset, ik, vmSymbols::isInCritical_name(), vmSymbols::bool_signature());
 }
 
 bool com_alibaba_wisp_engine_WispEngine::in_critical(oop obj) {
@@ -4773,9 +4773,8 @@ int com_alibaba_wisp_engine_WispTask::_stealCount_offset = 0;
 int com_alibaba_wisp_engine_WispTask::_stealFailureCount_offset = 0;
 
 void com_alibaba_wisp_engine_WispTask::compute_offsets() {
-  Klass* k = vmClasses::com_alibaba_wisp_engine_WispTask_klass();
-  assert(k != NULL, "WispTask_klass is null");
-  InstanceKlass *ik = InstanceKlass::cast(k);
+  InstanceKlass* ik = vmClasses::com_alibaba_wisp_engine_WispTask_klass();
+  assert(ik != NULL, "WispTask_klass is null");
   compute_offset(_jvmParkStatus_offset, ik, vmSymbols::jvmParkStatus_name(),   vmSymbols::int_signature());
   compute_offset(_id_offset,            ik, vmSymbols::id_name(),              vmSymbols::int_signature());
   compute_offset(_threadWrapper_offset, ik, vmSymbols::threadWrapper_name(),   vmSymbols::thread_signature());
diff --git a/src/hotspot/share/classfile/systemDictionary.cpp b/src/hotspot/share/classfile/systemDictionary.cpp
index cf6a44483a1..4d0cee4f0d4 100644
--- a/src/hotspot/share/classfile/systemDictionary.cpp
+++ b/src/hotspot/share/classfile/systemDictionary.cpp
@@ -224,7 +224,7 @@ Symbol* SystemDictionary::class_name_symbol(const char* name, Symbol* exception,
 #ifdef ASSERT
 // Used to verify that class loading succeeded in adding k to the dictionary.
 void verify_dictionary_entry(Symbol* class_name, InstanceKlass* k) {
-  SystemDictLocker mu(SystemDictionary_lock);
+  SystemDictLocker mu(JavaThread::current(), SystemDictionary_lock);
   ClassLoaderData* loader_data = k->class_loader_data();
   Dictionary* dictionary = loader_data->dictionary();
   assert(class_name == k->name(), "Must be the same");
@@ -470,7 +470,7 @@ InstanceKlass* SystemDictionary::resolve_super_or_fail(Symbol* class_name,
 //
 // The notify allows applications that did an untimed wait() on
 // the classloader object lock to not hang.
-static void double_lock_wait(SystemDictLocker *mu, JavaThread* thread, Handle lockObject) {
+static void double_lock_wait(JavaThread* thread, SystemDictLocker *mu, Handle lockObject) {
   assert_lock_strong(SystemDictionary_lock);
 
   assert(lockObject() != NULL, "lockObject must be non-NULL");
@@ -520,7 +520,8 @@ InstanceKlass* SystemDictionary::handle_parallel_loading(JavaThread* current,
                                                          Symbol* name,
                                                          ClassLoaderData* loader_data,
                                                          Handle lockObject,
-                                                         bool* throw_circularity_error) {
+                                                         bool* throw_circularity_error,
+                                                         SystemDictLocker* mu) {
   PlaceholderEntry* oldprobe = placeholders()->get_entry(name_hash, name, loader_data);
   if (oldprobe != NULL) {
     // only need check_seen_thread once, not on each loop
@@ -549,11 +550,10 @@ InstanceKlass* SystemDictionary::handle_parallel_loading(JavaThread* current,
         // which we will find below in the systemDictionary.
         oldprobe = NULL;  // Other thread could delete this placeholder entry
 
-        SystemDictLocker mu(JavaThread::current(), SystemDictionary_lock);
         if (lockObject.is_null()) {
-          mu.wait();
+          mu->wait();
         } else {
-          double_lock_wait(&mu, current, lockObject);
+          double_lock_wait(current, mu, lockObject);
         }
 
         // Check if classloading completed while we were waiting
@@ -689,7 +689,8 @@ InstanceKlass* SystemDictionary::resolve_instance_class_or_null(Symbol* name,
                                                name,
                                                loader_data,
                                                lockObject,
-                                               &throw_circularity_error);
+                                               &throw_circularity_error,
+                                               &mu);
       }
 
       // Recheck if the class has been loaded for all class loader cases and
@@ -732,7 +733,7 @@ InstanceKlass* SystemDictionary::resolve_instance_class_or_null(Symbol* name,
       // and initiating loaders
       SystemDictLocker mu(THREAD, SystemDictionary_lock);
       placeholders()->find_and_remove(name_hash, name, loader_data, PlaceholderTable::LOAD_INSTANCE, THREAD);
-      SystemDictionary_lock->notify_all(THREAD);
+      mu.notify_all();
     }
   }
 
@@ -1514,7 +1515,7 @@ InstanceKlass* SystemDictionary::find_or_define_helper(Symbol* class_name, Handl
     if (is_parallelDefine(class_loader) && (probe->instance_klass() != NULL)) {
       InstanceKlass* ik = probe->instance_klass();
       placeholders()->find_and_remove(name_hash, name_h, loader_data, PlaceholderTable::DEFINE_CLASS, THREAD);
-      SystemDictionary_lock->notify_all(THREAD);
+      mu.notify_all();
 #ifdef ASSERT
       InstanceKlass* check = dictionary->find_class(name_hash, name_h);
       assert(check != NULL, "definer missed recording success");
@@ -1538,7 +1539,7 @@ InstanceKlass* SystemDictionary::find_or_define_helper(Symbol* class_name, Handl
     }
     probe->set_definer(NULL);
     placeholders()->find_and_remove(name_hash, name_h, loader_data, PlaceholderTable::DEFINE_CLASS, THREAD);
-    SystemDictionary_lock->notify_all(THREAD);
+    mu.notify_all();
   }
 
   return HAS_PENDING_EXCEPTION ? NULL : k;
@@ -1606,7 +1607,7 @@ bool SystemDictionary::do_unloading(GCTimer* gc_timer) {
       MutexLocker ml2(is_concurrent ? Module_lock : NULL);
       JFR_ONLY(Jfr::on_unloading_classes();)
 
-      SystemDictLocker ml1(JavaThread::current(), is_concurrent ? SystemDictionary_lock : NULL);
+      GCSystemDictLocker ml1(is_concurrent ? SystemDictionary_lock : NULL);
       ClassLoaderDataGraph::clean_module_and_package_info();
       constraints()->purge_loader_constraints();
       resolution_errors()->purge_resolution_errors();
@@ -1777,7 +1778,7 @@ Klass* SystemDictionary::find_constrained_instance_or_array_klass(
     if (t != T_OBJECT) {
       klass = Universe::typeArrayKlassObj(t);
     } else {
-      SystemDictLocker mu(JavaThread::current(), SystemDictionary_lock);
+      SystemDictLocker mu((JavaThread*)current, SystemDictionary_lock);
       klass = constraints()->find_constrained_klass(ss.as_symbol(), class_loader);
     }
     // If element class already loaded, allocate array klass
@@ -1785,7 +1786,7 @@ Klass* SystemDictionary::find_constrained_instance_or_array_klass(
       klass = klass->array_klass_or_null(ndims);
     }
   } else {
-    SystemDictLocker mu(JavaThread::current(), SystemDictionary_lock);
+    SystemDictLocker mu((JavaThread*)current, SystemDictionary_lock);
     // Non-array classes are easy: simply check the constraint table.
     klass = constraints()->find_constrained_klass(class_name, class_loader);
   }
@@ -2488,8 +2489,3 @@ void SystemDictionaryDCmd::execute(DCmdSource source, TRAPS) {
                          _verbose.value());
   VMThread::execute(&dumper);
 }
-
-void SystemDictionary::system_dict_lock_change(JavaThread* THREAD) {
-  assert(UseWispMonitor, "UseWispMonitor is off");
-  SystemDictionary_lock->set_obj_lock(oopFactory::new_intArray(0, THREAD),  THREAD);
-}
diff --git a/src/hotspot/share/classfile/systemDictionary.hpp b/src/hotspot/share/classfile/systemDictionary.hpp
index 11dd5dc081a..787ea5baa51 100644
--- a/src/hotspot/share/classfile/systemDictionary.hpp
+++ b/src/hotspot/share/classfile/systemDictionary.hpp
@@ -334,7 +334,8 @@ class SystemDictionary : AllStatic {
                                                 Symbol* name,
                                                 ClassLoaderData* loader_data,
                                                 Handle lockObject,
-                                                bool* throw_circularity_error);
+                                                bool* throw_circularity_error,
+                                                SystemDictLocker *mu);
 
   static void define_instance_class(InstanceKlass* k, Handle class_loader, TRAPS);
   static InstanceKlass* find_or_define_helper(Symbol* class_name,
@@ -416,9 +417,6 @@ class SystemDictionary : AllStatic {
   static TableStatistics placeholders_statistics();
   static TableStatistics loader_constraints_statistics();
   static TableStatistics protection_domain_cache_statistics();
-
-public:
-  static void system_dict_lock_change(TRAPS);
 };
 
 #endif // SHARE_CLASSFILE_SYSTEMDICTIONARY_HPP
diff --git a/src/hotspot/share/gc/shared/barrierSet.cpp b/src/hotspot/share/gc/shared/barrierSet.cpp
index 614011827f4..c50b192e639 100644
--- a/src/hotspot/share/gc/shared/barrierSet.cpp
+++ b/src/hotspot/share/gc/shared/barrierSet.cpp
@@ -25,6 +25,7 @@
 #include "precompiled.hpp"
 #include "gc/shared/barrierSet.hpp"
 #include "gc/shared/barrierSetAssembler.hpp"
+#include "runtime/coroutine.hpp"
 #include "runtime/thread.hpp"
 #include "utilities/debug.hpp"
 #include "utilities/macros.hpp"
@@ -47,6 +48,9 @@ void BarrierSet::set_barrier_set(BarrierSet* barrier_set) {
   assert(!JavaThread::current()->on_thread_list(),
          "Main thread already on thread list.");
   _barrier_set->on_thread_create(Thread::current());
+  if (UseWispMonitor) {
+    _barrier_set->on_thread_create(WispThread::current(Thread::current()));
+  }
 }
 
 // Called from init.cpp
diff --git a/src/hotspot/share/interpreter/interpreterRuntime.cpp b/src/hotspot/share/interpreter/interpreterRuntime.cpp
index e9d68d12b77..03aff695190 100644
--- a/src/hotspot/share/interpreter/interpreterRuntime.cpp
+++ b/src/hotspot/share/interpreter/interpreterRuntime.cpp
@@ -590,6 +590,11 @@ JRT_ENTRY(void, InterpreterRuntime::throw_pending_exception(JavaThread* current)
   // nothing to do - eventually we should remove this code entirely (see comments @ call sites)
 JRT_END
 
+#ifdef ASSERT
+JRT_ENTRY(void, InterpreterRuntime::print_site(JavaThread* current, void* arg0, void* arg1))
+  assert(current->has_pending_exception(), "must only be called if there's an exception pending");
+JRT_END
+#endif
 
 JRT_ENTRY(void, InterpreterRuntime::throw_AbstractMethodError(JavaThread* current))
   THROW(vmSymbols::java_lang_AbstractMethodError());
@@ -732,10 +737,9 @@ JRT_ENTRY_NO_ASYNC(void, InterpreterRuntime::monitorenter(JavaThread* current, B
   if (PrintBiasedLockingStatistics) {
     Atomic::inc(BiasedLocking::slow_path_entry_count_addr());
   }
-  
-  Thread* t = THREAD;
+
   // thread steal support
-  WispPostStealHandleUpdateMark w(current, t, __tiv, __hm);
+  WispPostStealHandleUpdateMark w(current, (Thread *&)THREAD, __tiv, __hm);
 
   // Coroutine work steal support
   EnableStealMark p(THREAD);
diff --git a/src/hotspot/share/interpreter/interpreterRuntime.hpp b/src/hotspot/share/interpreter/interpreterRuntime.hpp
index c32431784aa..5e113b233c8 100644
--- a/src/hotspot/share/interpreter/interpreterRuntime.hpp
+++ b/src/hotspot/share/interpreter/interpreterRuntime.hpp
@@ -94,6 +94,11 @@ class InterpreterRuntime: AllStatic {
   static void    throw_pending_exception(JavaThread* current);
 
   static void resolve_from_cache(JavaThread* current, Bytecodes::Code bytecode);
+
+#ifdef ASSERT
+  static void print_site(JavaThread* current, void* arg0, void* arg1);
+#endif
+
  private:
   // Statics & fields
   static void resolve_get_put(JavaThread* current, Bytecodes::Code bytecode);
diff --git a/src/hotspot/share/jvmci/jvmciRuntime.cpp b/src/hotspot/share/jvmci/jvmciRuntime.cpp
index aca748d2442..4953f324f34 100644
--- a/src/hotspot/share/jvmci/jvmciRuntime.cpp
+++ b/src/hotspot/share/jvmci/jvmciRuntime.cpp
@@ -392,7 +392,7 @@ address JVMCIRuntime::exception_handler_for_pc(JavaThread* current) {
 }
 
 JRT_BLOCK_ENTRY(void, JVMCIRuntime::monitorenter(JavaThread* current, oopDesc* obj, BasicLock* lock))
-WispPostStealHandleUpdateMark w(__hm);
+  WispPostStealHandleUpdateMark w(__hm);
   SharedRuntime::monitor_enter_helper(obj, lock, current);
 JRT_END
 
diff --git a/src/hotspot/share/oops/weakHandle.hpp b/src/hotspot/share/oops/weakHandle.hpp
index c2b589958d2..df6a9fef37f 100644
--- a/src/hotspot/share/oops/weakHandle.hpp
+++ b/src/hotspot/share/oops/weakHandle.hpp
@@ -26,7 +26,7 @@
 #define SHARE_OOPS_WEAKHANDLE_HPP
 
 #include "oops/oop.hpp"
-#include "runtime/handles.hpp"
+#include "runtime/handles.inline.hpp"
 
 class outputStream;
 class OopStorage;
diff --git a/src/hotspot/share/opto/graphKit.cpp b/src/hotspot/share/opto/graphKit.cpp
index b50e7abb870..0a8552c7a29 100644
--- a/src/hotspot/share/opto/graphKit.cpp
+++ b/src/hotspot/share/opto/graphKit.cpp
@@ -2625,8 +2625,7 @@ Node* GraphKit::make_runtime_call(int flags,
                                   Node* parm0, Node* parm1,
                                   Node* parm2, Node* parm3,
                                   Node* parm4, Node* parm5,
-                                  Node* parm6, Node* parm7,
-                                  Node* ctrl) {
+                                  Node* parm6, Node* parm7) {
   assert(call_addr != NULL, "must not call NULL targets");
 
   // Slow-path call
@@ -2657,7 +2656,7 @@ Node* GraphKit::make_runtime_call(int flags,
 
   Node* prev_mem = NULL;
   if (wide_in) {
-    prev_mem = set_predefined_input_for_runtime_call(call, ctrl);
+    prev_mem = set_predefined_input_for_runtime_call(call);
   } else {
     assert(!wide_out, "narrow in => narrow out");
     Node* narrow_mem = memory(adr_type);
@@ -3663,9 +3662,7 @@ void GraphKit::make_wisp_yield(ciMethod* method) {
   Node* mark_preempt_addr = __ AddP(no_base, tls, __ ConX(mark_offset));
   Node* mark_preempt_val = __ load(__ ctrl(), mark_preempt_addr, TypeInt::BYTE, T_BYTE, Compile::AliasIdxRaw);
   Node*   zero = __ ConI(0);
-  const TypeFunc *call_type    = OptoRuntime::yield_method_exit_Type();
-  address         call_address = CAST_FROM_FN_PTR(address, SharedRuntime::wisp_yield);
-  const char     *call_name    = "wisp_yield";
+
   // Get base of thread-local storage area
   Node* thread = _gvn.transform( new ThreadLocalNode() );
 
@@ -3676,17 +3673,16 @@ void GraphKit::make_wisp_yield(ciMethod* method) {
   kill_dead_locals();
 
   // For some reason, this call reads only raw memory.
-  const TypePtr* raw_adr_type = TypeRawPtr::BOTTOM;
   __ if_then(mark_preempt_val, BoolTest::ne, zero) ; {
-    // Totally 8 parameters, 2 used, 6 NULL
+    // Totally 8 parameters, 2 used, 6 NULL.
+    // make_runtime_call itself would set control node, so
+    // we should never pass control node here.
     Node* call = make_runtime_call(RC_NARROW_MEM,
-                    call_type, call_address,
-                    call_name, raw_adr_type,
-                    thread, method_node, 
-                    NULL, NULL, NULL,
-                    NULL, NULL, NULL,
-                    __ ctrl());
-    __ set_ctrl( _gvn.transform( new ProjNode(call, TypeFunc::Control)));
+                    OptoRuntime::yield_method_exit_Type(),
+                    CAST_FROM_FN_PTR(address, SharedRuntime::wisp_yield),
+                    "wisp_yield",
+                    TypeRawPtr::BOTTOM,
+                    thread, method_node);
   } __ end_if();
   final_sync(ideal);
 }
diff --git a/src/hotspot/share/opto/graphKit.hpp b/src/hotspot/share/opto/graphKit.hpp
index 422adf1c23e..1ab68efe8e8 100644
--- a/src/hotspot/share/opto/graphKit.hpp
+++ b/src/hotspot/share/opto/graphKit.hpp
@@ -803,8 +803,7 @@ class GraphKit : public Phase {
                           Node* parm0 = NULL, Node* parm1 = NULL,
                           Node* parm2 = NULL, Node* parm3 = NULL,
                           Node* parm4 = NULL, Node* parm5 = NULL,
-                          Node* parm6 = NULL, Node* parm7 = NULL,
-                          Node* ctrl = NULL);
+                          Node* parm6 = NULL, Node* parm7 = NULL);
 
   void make_wisp_yield(ciMethod* method);
   Node* sign_extend_byte(Node* in);
diff --git a/src/hotspot/share/opto/library_call.cpp b/src/hotspot/share/opto/library_call.cpp
index b5970545c4e..c84fc886db5 100644
--- a/src/hotspot/share/opto/library_call.cpp
+++ b/src/hotspot/share/opto/library_call.cpp
@@ -870,9 +870,35 @@ Node* LibraryCallKit::generate_current_thread(Node* &tls_output) {
   Node* thread = _gvn.transform(new ThreadLocalNode());
   Node* p = basic_plus_adr(top()/*!oop*/, thread, in_bytes(JavaThread::threadObj_offset()));
   tls_output = thread;
-  Node* thread_obj_handle = LoadNode::make(_gvn, NULL, immutable_memory(), p, p->bottom_type()->is_ptr(), TypeRawPtr::NOTNULL, T_ADDRESS, MemNode::unordered);
+
+  Node* thread_obj_handle = NULL;
+  DecoratorSet decorators = IN_NATIVE;
+  // threadObj oop is linked to JavaThread by oopstorage at Java17 while it is
+  // linked to JavaThread as oop at java11. Java11 always refetches threadObj
+  // because oop may be moved by GC. In theory, Java17 also should always
+  // refetch threadObj. But threadObj refetch by oopstorage is a little
+  // expensive because oopstorage needs one more dereference. To save cost,
+  // Java17 current_thread inline caches the oopstorage address because
+  // oopstorage address is never changed during the method in thread semantic.
+  // threadObj oop refech can be speed up if oopstorage address is cached
+  // because we need not to refetch oopstorage address from r15_thread. We
+  // can get threadObj directly from the cached oopstorage address.
+  // In the middle of method execution, the underly thread is never changed and
+  // oopstorage caching is a good optimization for thread semantic.
+  // But in coroutine semantic, one method may be executed by different
+  // underly threads. The oopstorage caching may lead to wrong results
+  // because of coroutine stealing machanism. So in coroutine mode, we should
+  // never cache oopstorage to speed up threadObj refetch.
+  if (EnableCoroutine) {
+    // if enable coroutine, we should use make_load to avoid threadObj changed
+    thread_obj_handle = make_load(nullptr, p, p->bottom_type()->is_ptr(), T_ADDRESS, MemNode::unordered);
+  } else {
+    thread_obj_handle = LoadNode::make(_gvn, NULL, immutable_memory(), p, p->bottom_type()->is_ptr(), TypeRawPtr::NOTNULL, T_ADDRESS, MemNode::unordered);
+    decorators |= C2_IMMUTABLE_MEMORY;
+  }
+
   thread_obj_handle = _gvn.transform(thread_obj_handle);
-  return access_load(thread_obj_handle, thread_type, T_OBJECT, IN_NATIVE | C2_IMMUTABLE_MEMORY);
+  return access_load(thread_obj_handle, thread_type, T_OBJECT, decorators);
 }
 
 
diff --git a/src/hotspot/share/prims/jvm.cpp b/src/hotspot/share/prims/jvm.cpp
index 1f48f7fa299..beaf32c6521 100644
--- a/src/hotspot/share/prims/jvm.cpp
+++ b/src/hotspot/share/prims/jvm.cpp
@@ -607,8 +607,7 @@ JVM_ENTRY(void, JVM_MonitorWait(JNIEnv* env, jobject handle, jlong ms))
   Handle obj(THREAD, JNIHandles::resolve_non_null(handle));
   JavaThreadInObjectWaitState jtiows(thread, ms != 0);
 
-  Thread* t = THREAD;
-  WispPostStealHandleUpdateMark w(thread, t, env, __tiv, __hm, &jtiows);
+  WispPostStealHandleUpdateMark w(thread, (Thread *&)THREAD, env, __tiv, __hm, &jtiows);
 
   // Coroutine work steal support
   EnableStealMark p(THREAD);
@@ -3119,7 +3118,7 @@ JVM_ENTRY(jboolean, JVM_CheckAndClearNativeInterruptForWisp(JNIEnv* env, jobject
   assert(EnableCoroutine, "Coroutine is disabled");
   JavaThread *th = java_lang_Thread::thread(JNIHandles::resolve_non_null(jthread));
   if (th != NULL) {
-    return (jboolean)clear_interrupt_for_wisp(th);
+    return (jboolean)(th->clear_interrupt_for_wisp());
   } else {
     return (jboolean)false;
   }
@@ -3514,8 +3513,7 @@ jclass find_class_from_class_loader(JNIEnv* env, Symbol* name, jboolean init,
 
 JVM_ENTRY(jobject, JVM_InvokeMethod(JNIEnv *env, jobject method, jobject obj, jobjectArray args0))
   // Coroutine work steal support
-  Thread* t = THREAD;
-  WispPostStealHandleUpdateMark w(thread, t, env, __tiv, __hm);
+  WispPostStealHandleUpdateMark w(thread, (Thread *&)THREAD, env, __tiv, __hm);
 
   Handle method_handle;
   if (thread->stack_overflow_state()->stack_available((address) &method_handle) >= JVMInvokeMethodSlack) {
@@ -3542,8 +3540,7 @@ JVM_END
 
 JVM_ENTRY(jobject, JVM_NewInstanceFromConstructor(JNIEnv *env, jobject c, jobjectArray args0))
   // Coroutine work steal support
-  Thread* t = THREAD;
-  WispPostStealHandleUpdateMark w(thread, t, env, __tiv, __hm);
+  WispPostStealHandleUpdateMark w(thread, (Thread *&)THREAD, env, __tiv, __hm);
 
   oop constructor_mirror = JNIHandles::resolve(c);
   objArrayHandle args(THREAD, objArrayOop(JNIHandles::resolve(args0)));
diff --git a/src/hotspot/share/prims/universalUpcallHandler.cpp b/src/hotspot/share/prims/universalUpcallHandler.cpp
index 44b64504d1c..993b7a03a6b 100644
--- a/src/hotspot/share/prims/universalUpcallHandler.cpp
+++ b/src/hotspot/share/prims/universalUpcallHandler.cpp
@@ -108,7 +108,7 @@ JavaThread* ProgrammableUpcallHandler::on_entry(OptimizedEntryBlob::FrameData* c
   context->jfa.copy(thread->frame_anchor());
   thread->frame_anchor()->clear();
 
-  debug_only(thread->inc_java_call_counter());
+  thread->inc_java_call_counter();
   thread->set_active_handles(context->new_handles);     // install new handle block and reset Java frame linkage
 
   // clear any pending exception in thread (native calls start with no exception pending)
@@ -133,7 +133,7 @@ void ProgrammableUpcallHandler::on_exit(OptimizedEntryBlob::FrameData* context)
 
   thread->frame_anchor()->zap();
 
-  debug_only(thread->dec_java_call_counter());
+  thread->dec_java_call_counter();
 
   // Old thread-local info. has been restored. We are now back in native code.
   ThreadStateTransition::transition_from_java(thread, _thread_in_native);
diff --git a/src/hotspot/share/runtime/coroutine.cpp b/src/hotspot/share/runtime/coroutine.cpp
index de6514612b7..640595e616d 100644
--- a/src/hotspot/share/runtime/coroutine.cpp
+++ b/src/hotspot/share/runtime/coroutine.cpp
@@ -23,7 +23,6 @@
  */
 
 #include "precompiled.hpp"
-//#include "prims/privilegedStack.hpp"
 #include "classfile/vmSymbols.hpp"
 #include "interpreter/linkResolver.hpp"
 #include "runtime/coroutine.hpp"
@@ -128,7 +127,6 @@ Coroutine* Coroutine::create_thread_coroutine(JavaThread* thread, CoroutineStack
 #if defined(_WINDOWS)
   coro->_last_SEH = NULL;
 #endif
-  // coro->_privileged_stack_top = NULL;
   coro->_wisp_thread  = UseWispMonitor ? new WispThread(coro) : NULL;
   coro->_wisp_engine  = NULL;
   coro->_wisp_task    = NULL;
@@ -177,7 +175,6 @@ Coroutine* Coroutine::create_coroutine(JavaThread* thread, CoroutineStack* stack
 #if defined(_WINDOWS)
   coro->_last_SEH = NULL;
 #endif
-  // coro->_privileged_stack_top = NULL;
   coro->_wisp_thread  = UseWispMonitor ? new WispThread(coro) : NULL;
   coro->_wisp_engine  = NULL;
   coro->_wisp_task    = NULL;
@@ -246,9 +243,6 @@ void Coroutine::oops_do(OopClosure* f, CodeBlobClosure* cf) {
     DEBUG_CORO_ONLY(tty->print_cr("collecting handle area %08x", _handle_area));
     _handle_area->oops_do(f);
     _active_handles->oops_do(f);
-    // if (_privileged_stack_top != NULL) {
-    //   _privileged_stack_top->oops_do(f);
-    // } 
   }
   if (_wisp_task != NULL) {
     f->do_oop((oop*) &_wisp_engine);
@@ -332,7 +326,7 @@ CoroutineStack* CoroutineStack::create_stack(JavaThread* thread, intptr_t size/*
     default_size = true;
   }
 
-  uint reserved_pages = StackShadowPages + StackRedPages + StackYellowPages + + StackReservedPages;
+  uint reserved_pages = StackShadowPages + StackRedPages + StackYellowPages + StackReservedPages;
   uintx real_stack_size = size + (reserved_pages * os::vm_page_size());
   uintx reserved_size = align_up(real_stack_size, os::vm_allocation_granularity());
 
@@ -475,17 +469,17 @@ void WispThread::set_wisp_booted(JavaThread* thread) {
   LinkInfo link_info(vmClasses::com_alibaba_wisp_engine_WispTask_klass(), vmSymbols::park_name(), vmSymbols::long_void_signature());
   LinkResolver::resolve_static_call(callinfo, link_info, true, thread);
   parkMethod = callinfo.selected_method();
-  assert(parkMethod.not_null(), "should have thrown exception");
+  assert(parkMethod != NULL, "should have thrown exception");
 
   LinkInfo link_info_unpark(vmClasses::com_alibaba_wisp_engine_WispTask_klass(), vmSymbols::unparkById_name(), vmSymbols::int_void_signature());
   LinkResolver::resolve_static_call(callinfo, link_info_unpark, true, thread);
   unparkMethod = callinfo.selected_method();
-  assert(unparkMethod.not_null(), "should have thrown exception");
+  assert(unparkMethod != NULL, "should have thrown exception");
 
   if (UseWispMonitor) {
-    _proxy_unpark = new (ResourceObj::C_HEAP, mtWisp) GrowableArray(30);
+    _proxy_unpark = new (ResourceObj::C_HEAP, mtWisp) GrowableArray(30, mtWisp);
     if (!AlwaysLockClassLoader) {
-      SystemDictionary::system_dict_lock_change(thread);
+      SystemDictionary_lock->create_obj_lock(thread);
     }
   }
 
@@ -681,9 +675,14 @@ void WispThread::unpark(int task_id, bool using_wisp_park, bool proxy_unpark, Pa
 }
 
 int WispThread::get_proxy_unpark(jintArray res) {
+  // We need to hoist code of safepoint state out of MutexLocker to prevent safepoint deadlock problem
+  // See the same usage: SR_lock in `JavaThread::exit()`
+  ThreadBlockInVM tbivm(JavaThread::current());
+  // When wait()ing, GC may occur. So we shouldn't verify GC.
+  NoSafepointVerifier nsv;
   MutexLocker mu(Wisp_lock, Mutex::_no_safepoint_check_flag);
   while (_proxy_unpark == NULL || _proxy_unpark->is_empty()) {
-    Wisp_lock->wait();
+    Wisp_lock->wait_without_safepoint_check();
   }
   typeArrayOop a = typeArrayOop(JNIHandles::resolve_non_null(res));
   if (a == NULL) {
@@ -892,6 +891,14 @@ WispPostStealHandleUpdateMark::WispPostStealHandleUpdateMark(JavaThread *thread,
   WispPostStealHandle h(&(tbv._tbivmpp));
 }
 
+WispPostStealHandleUpdateMark::WispPostStealHandleUpdateMark(JavaThread *thread, ThreadStateTransition & tst)
+{
+  initialize(thread, true);
+
+  if (!_success)  return;
+  WispPostStealHandle h(&tst);
+}
+
 WispPostStealHandleUpdateMark::WispPostStealHandleUpdateMark(JavaThread *&th)    // this is a special one
 {
   initialize(th, true);
diff --git a/src/hotspot/share/runtime/coroutine.hpp b/src/hotspot/share/runtime/coroutine.hpp
index 6bbbe1418ed..1b6c31e5b0d 100644
--- a/src/hotspot/share/runtime/coroutine.hpp
+++ b/src/hotspot/share/runtime/coroutine.hpp
@@ -49,6 +49,7 @@
 #define DEBUG_CORO_PRINT(x)
 #endif
 
+class ThreadStateTransition;
 class Coroutine;
 class CoroutineStack;
 class WispThread;
@@ -115,7 +116,6 @@ class Coroutine: public CHeapObj, public DoublyLinkedList {
   JNIHandleBlock* _active_handles;
   GrowableArray* _metadata_handles;
   JavaFrameAnchor _anchor;
-  // PrivilegedElement*  _privileged_stack_top;
   JavaThreadStatus _thread_status;
   int             _enable_steal_count;
   int             _java_call_counter;
@@ -208,7 +208,6 @@ class Coroutine: public CHeapObj, public DoublyLinkedList {
   static ByteSize last_handle_mark_offset()   { return byte_offset_of(Coroutine, _last_handle_mark); }
   static ByteSize active_handles_offset()     { return byte_offset_of(Coroutine, _active_handles); }
   static ByteSize metadata_handles_offset()   { return byte_offset_of(Coroutine, _metadata_handles); }
-  // static ByteSize privileged_stack_top_offset(){ return byte_offset_of(Coroutine, _privileged_stack_top); }
   static ByteSize last_Java_sp_offset()       {
     return byte_offset_of(Coroutine, _anchor) + JavaFrameAnchor::last_Java_sp_offset();
   }
@@ -525,6 +524,7 @@ class WispPostStealHandleUpdateMark: public StackObj {
                     ThreadInVMfromJava & tiva);
   WispPostStealHandleUpdateMark(HandleMarkCleaner & hmc);
   WispPostStealHandleUpdateMark(JavaThread *thread, ThreadBlockInVM & tbv);  // constructor is used inside objectMonitor call, which is also within EnableStealMark scope.
+  WispPostStealHandleUpdateMark(JavaThread *thread, ThreadStateTransition & tst);
   WispPostStealHandleUpdateMark(JavaThread *& th);                  // this is a special one, used for a fix inside EnableStealMark
 
   ~WispPostStealHandleUpdateMark();
diff --git a/src/hotspot/share/runtime/handles.hpp b/src/hotspot/share/runtime/handles.hpp
index b8ecd42c7fa..a7e09cd38c7 100644
--- a/src/hotspot/share/runtime/handles.hpp
+++ b/src/hotspot/share/runtime/handles.hpp
@@ -328,7 +328,6 @@ class HandleMarkCleaner: public StackObj {
   inline HandleMarkCleaner(Thread* thread);
   inline ~HandleMarkCleaner();
   Thread *& thread_ref() {
-    assert(EnableCoroutine, "EnableCoroutine is off");
     return _thread;
   }
 };
diff --git a/src/hotspot/share/runtime/javaCalls.cpp b/src/hotspot/share/runtime/javaCalls.cpp
index e78249924bc..f6031655bd5 100644
--- a/src/hotspot/share/runtime/javaCalls.cpp
+++ b/src/hotspot/share/runtime/javaCalls.cpp
@@ -39,6 +39,7 @@
 #include "prims/jniCheck.hpp"
 #include "prims/jvmtiExport.hpp"
 #include "runtime/handles.inline.hpp"
+#include "runtime/init.hpp"
 #include "runtime/interfaceSupport.inline.hpp"
 #include "runtime/javaCalls.hpp"
 #include "runtime/jniHandles.inline.hpp"
@@ -347,9 +348,6 @@ Handle JavaCalls::construct_new_instance(InstanceKlass* klass, Symbol* construct
 void JavaCalls::call(JavaValue* result, const methodHandle& method, JavaCallArguments* args, TRAPS) {
   // Check if we need to wrap a potential OS exception handler around thread.
   // This is used for e.g. Win32 structured exception handlers.
-  assert(!UseWispMonitor || !is_init_completed() ||
-      java_lang_Thread::park_event(((JavaThread*) THREAD)->threadObj()),
-      "park_event need to be set before calling java");
   // Need to wrap each and every time, since there might be native code down the
   // stack that has installed its own exception handlers.
   os::os_exception_wrapper(call_helper, result, method, args, THREAD);
@@ -440,8 +438,7 @@ void JavaCalls::call_helper(JavaValue* result, const methodHandle& method, JavaC
 #endif
       // thread steal support
       methodHandle* m = const_cast(&method);
-      Thread* t = THREAD;
-      WispPostStealHandleUpdateMark w(thread, t, *m, m, link);
+      WispPostStealHandleUpdateMark w(thread, (Thread *&)THREAD, *m, m, link);
 
       StubRoutines::call_stub()(
         (address)&link,
diff --git a/src/hotspot/share/runtime/mutexLocker.cpp b/src/hotspot/share/runtime/mutexLocker.cpp
index d9a9fc8bea0..acdab61c3eb 100644
--- a/src/hotspot/share/runtime/mutexLocker.cpp
+++ b/src/hotspot/share/runtime/mutexLocker.cpp
@@ -25,6 +25,7 @@
 #include "precompiled.hpp"
 #include "gc/shared/gc_globals.hpp"
 #include "memory/universe.hpp"
+#include "runtime/coroutine.hpp"
 #include "runtime/mutexLocker.hpp"
 #include "runtime/os.inline.hpp"
 #include "runtime/safepoint.hpp"
@@ -217,7 +218,6 @@ static bool is_owner(const SystemDictMonitor* lock, Thread* THREAD) {
 }
 
 void assert_lock_strong(const SystemDictMonitor* lock) {
-  if (IgnoreLockingAssertions) return;
   assert(lock != NULL, "Need non-NULL lock");
   if (is_owner(lock, Thread::current())) return;
   fatal("must own lock %s", lock->monitor()->name());
@@ -225,7 +225,6 @@ void assert_lock_strong(const SystemDictMonitor* lock) {
 
 void assert_locked_or_safepoint(const SystemDictMonitor* lock) {
   // check if this thread owns the lock (common case)
-  if (IgnoreLockingAssertions) return;
   assert(lock != NULL, "Need non-NULL lock");
   if (SafepointSynchronize::is_at_safepoint()) return;
   if (is_owner(lock, Thread::current())) return;
@@ -408,8 +407,11 @@ GCMutexLocker::GCMutexLocker(Mutex* mutex) {
   }
 }
 
+SystemDictLocker::SystemDictLocker(JavaThread* THREAD, SystemDictMonitor* mutex, bool do_lock)
+  : SystemDictLockerBase(THREAD, mutex, do_lock) {}
+
 GCSystemDictLocker::GCSystemDictLocker(SystemDictMonitor* mutex)
-  : SystemDictLocker(JavaThread::current(), mutex, !SafepointSynchronize::is_at_safepoint()) {}
+  : SystemDictLockerBase(Thread::current(), mutex, !SafepointSynchronize::is_at_safepoint()) {}
 
 // Print all mutexes/monitors that are currently owned by a thread; called
 // by fatal error handler.
diff --git a/src/hotspot/share/runtime/mutexLocker.hpp b/src/hotspot/share/runtime/mutexLocker.hpp
index 0893d940abe..aa9a5703efd 100644
--- a/src/hotspot/share/runtime/mutexLocker.hpp
+++ b/src/hotspot/share/runtime/mutexLocker.hpp
@@ -28,6 +28,7 @@
 #include "memory/allocation.hpp"
 #include "runtime/flags/flagSetting.hpp"
 #include "runtime/mutex.hpp"
+#include "runtime/synchronizer.hpp"
 
 // Mutexes used in the VM.
 
@@ -320,18 +321,19 @@ class MutexUnlocker: StackObj {
   }
 };
 
-class SystemDictLocker: StackObj {
+
+class SystemDictLockerBase: StackObj {
  private:
   SystemDictMonitor* _mutex;
-  JavaThread*        _thread;
+  Thread*            _thread;
   BasicLock          _lock;
   bool               _locked;
  public:
-  SystemDictLocker(JavaThread* THREAD, SystemDictMonitor* mutex, bool do_lock=true) {
+  SystemDictLockerBase(Thread* thread, SystemDictMonitor* mutex, bool do_lock=true) {
     _locked = do_lock;
     _mutex = mutex;
-    _thread = THREAD;
-    if (do_lock) {
+    _thread = thread;
+    if (_locked && _mutex != NULL) {
       _mutex->lock(&_lock, _thread);
     }
   }
@@ -342,10 +344,15 @@ class SystemDictLocker: StackObj {
   void unlock()     { _mutex->unlock(&_lock, _thread); }
 
 
-  ~SystemDictLocker() { if (_locked)  _mutex->unlock(&_lock, _thread); }
+  ~SystemDictLockerBase() { if (_locked && _mutex != NULL)  _mutex->unlock(&_lock, _thread); }
+};
+
+class SystemDictLocker: public SystemDictLockerBase {
+ public:
+  SystemDictLocker(JavaThread* THREAD, SystemDictMonitor* mutex, bool do_lock=true);
 };
 
-class GCSystemDictLocker: public SystemDictLocker {
+class GCSystemDictLocker: public SystemDictLockerBase {
  public:
   GCSystemDictLocker(SystemDictMonitor* mutex);
 };
diff --git a/src/hotspot/share/runtime/objectMonitor.cpp b/src/hotspot/share/runtime/objectMonitor.cpp
index e404201725b..464cbff3381 100644
--- a/src/hotspot/share/runtime/objectMonitor.cpp
+++ b/src/hotspot/share/runtime/objectMonitor.cpp
@@ -422,7 +422,12 @@ bool ObjectMonitor::enter(JavaThread* current) {
     for (;;) {
       ExitOnSuspend eos(this);
       {
-        ThreadBlockInVMPreprocess tbivs(current, eos);
+        // If UseWispMonitor, We should record the stack frame and Thread status onto the
+        // JavaThread which corresponds to the WispThread, instead of WispThread itself.
+        ThreadBlockInVMPreprocess tbivs(UseWispMonitor ? ((WispThread*)current)->thread(): current, eos);
+
+        // Coroutine work steal support
+        WispPostStealHandleUpdateMark w(UseWispMonitor ? ((WispThread*)current)->thread() : current, (ThreadStateTransition &)tbivs);
         EnterI(current);
         current->set_current_pending_monitor(NULL);
         // We can go to a safepoint at the end of this block. If we
@@ -689,7 +694,9 @@ void ObjectMonitor::EnterI(JavaThread* current) {
   if (UseWispMonitor) {
     current = WispThread::current(current);
   }
-  assert(current->thread_state() == _thread_blocked, "invariant");
+  assert(current->thread_state() == _thread_blocked ||
+    UseWispMonitor && ((WispThread*) current)->thread()->thread_state() == _thread_blocked,
+    "invariant");
 
   // Try the lock - TATAS
   if (TryLock (current) > 0) {
@@ -987,9 +994,12 @@ void ObjectMonitor::ReenterI(JavaThread* current, ObjectWaiter* currentNode) {
 
       {
         ClearSuccOnSuspend csos(this);
-        ThreadBlockInVMPreprocess tbivs(current, csos);
+        ThreadBlockInVMPreprocess tbivs(UseWispMonitor ? ((WispThread *)current)->thread() : current, csos);
+
+        // Coroutine work steal support
+        WispPostStealHandleUpdateMark w(UseWispMonitor ? ((WispThread*)current)->thread() : current, (ThreadStateTransition &)tbivs);
         if (UseWispMonitor) {
-          WispThread::park(MAX_RECHECK_INTERVAL, currentNode);
+          WispThread::park(-1, currentNode);
         } else {
           current->_ParkEvent->park();
         }
@@ -1451,10 +1461,10 @@ bool ObjectMonitor::reenter(intx recursions, JavaThread* current) {
 // (IMSE). If there is a pending exception and the specified thread
 // is not the owner, that exception will be replaced by the IMSE.
 bool ObjectMonitor::check_owner(TRAPS) {
+  JavaThread* current = THREAD;
   if (UseWispMonitor) {
-    THREAD = WispThread::current(THREAD);
+    current = WispThread::current(current);
   }
-  JavaThread* current = THREAD;
   void* cur = owner_raw();
   if (cur == current) {
     return true;
@@ -1496,12 +1506,14 @@ static void post_monitor_wait_event(EventJavaMonitorWait* event,
   event->commit();
 }
 
-static bool check_interrupt(Thread* self, bool clear_interrupted) {
+// is_interrupted method is not virtual.
+// So we should dispatch it by type convertion.
+static bool check_interrupt(Thread* current, bool clear_interrupted) {
   if (UseWispMonitor) {
-    assert(self->is_Wisp_thread(), "must be");
-    return ((WispThread*) self)->is_interrupted(clear_interrupted);
+    assert(current->is_Wisp_thread(), "must be");
+    return ((WispThread*) current)->is_interrupted(clear_interrupted);
   } else {
-    return ((JavaThread*) self)->is_interrupted(clear_interrupted);
+    return ((JavaThread*) current)->is_interrupted(clear_interrupted);
   }
 }
 
@@ -1511,10 +1523,10 @@ static bool check_interrupt(Thread* self, bool clear_interrupted) {
 // Note: a subset of changes to ObjectMonitor::wait()
 // will need to be replicated in complete_exit
 void ObjectMonitor::wait(jlong millis, bool interruptible, TRAPS) {
+  JavaThread* current = THREAD;
   if (UseWispMonitor) {
-    THREAD = WispThread::current(THREAD);
+    current = WispThread::current(current);
   }
-  JavaThread* current = THREAD;
 
   assert(InitDone, "Unexpectedly not initialized");
 
@@ -1545,7 +1557,7 @@ void ObjectMonitor::wait(jlong millis, bool interruptible, TRAPS) {
     return;
   }
 
-  assert(current->_Stalled || UseWispMonitor == 0, "invariant");
+  assert(current->_Stalled == 0 || UseWispMonitor, "invariant");
   current->_Stalled = intptr_t(this);
   current->set_current_waiting_monitor(this);
 
@@ -1591,7 +1603,7 @@ void ObjectMonitor::wait(jlong millis, bool interruptible, TRAPS) {
   int WasNotified = 0;
 
   // Need to check interrupt state whilst still _thread_in_vm
-  bool interrupted = interruptible && current->is_interrupted(false);
+  bool interrupted = interruptible && check_interrupt(current, false);
 
   { // State transition wrappers
     OSThread* osthread = current->osthread();
@@ -1601,7 +1613,11 @@ void ObjectMonitor::wait(jlong millis, bool interruptible, TRAPS) {
 
     {
       ClearSuccOnSuspend csos(this);
-      ThreadBlockInVMPreprocess tbivs(current, csos);
+      ThreadBlockInVMPreprocess tbivs(UseWispMonitor ? ((WispThread*)current)->thread() : current, csos);
+
+      // Coroutine work steal support
+      WispPostStealHandleUpdateMark w(UseWispMonitor ? ((WispThread*)current)->thread() : current, (ThreadStateTransition &)tbivs);
+
       if (interrupted || HAS_PENDING_EXCEPTION) {
         // Intentionally empty
       } else if (node._notified == 0) {
@@ -1811,10 +1827,10 @@ void ObjectMonitor::INotify(JavaThread* current) {
 // that suggests a lost wakeup bug.
 
 void ObjectMonitor::notify(TRAPS) {
+  JavaThread* current = THREAD;
   if (UseWispMonitor) {
-    THREAD = WispThread::current(THREAD);
+    current = WispThread::current(current);
   }
-  JavaThread* current = THREAD;
   CHECK_OWNER();  // Throws IMSE if not owner.
   if (_WaitSet == NULL) {
     return;
@@ -1833,10 +1849,10 @@ void ObjectMonitor::notify(TRAPS) {
 // mode the waitset will be empty and the EntryList will be "DCBAXYZ".
 
 void ObjectMonitor::notifyAll(TRAPS) {
+  JavaThread* current = THREAD;
   if (UseWispMonitor) {
-    THREAD = WispThread::current(THREAD);
+    current = WispThread::current(current);
   }
-  JavaThread* current = THREAD;
   CHECK_OWNER();  // Throws IMSE if not owner.
   if (_WaitSet == NULL) {
     return;
diff --git a/src/hotspot/share/runtime/os.hpp b/src/hotspot/share/runtime/os.hpp
index 5bb451c9e25..b0d1ad9ae48 100644
--- a/src/hotspot/share/runtime/os.hpp
+++ b/src/hotspot/share/runtime/os.hpp
@@ -1039,6 +1039,4 @@ class os: AllStatic {
 
 extern "C" int SpinPause();
 
-bool clear_interrupt_for_wisp(Thread *);
-
 #endif // SHARE_RUNTIME_OS_HPP
diff --git a/src/hotspot/share/runtime/reflection.cpp b/src/hotspot/share/runtime/reflection.cpp
index 5ef52d50450..f67c0c6c172 100644
--- a/src/hotspot/share/runtime/reflection.cpp
+++ b/src/hotspot/share/runtime/reflection.cpp
@@ -1108,8 +1108,7 @@ static oop invoke(InstanceKlass* klass,
   JavaValue result(rtype);
 
   {
-    Thread* t = THREAD;
-    WispPostStealHandleUpdateMark w(t, const_cast(&reflected_method), &method);
+    WispPostStealHandleUpdateMark w((Thread *&)THREAD, const_cast(&reflected_method), &method);
     EnableStealMark p(THREAD);
     JavaCalls::call(&result, method, &java_args, THREAD);
   }
@@ -1159,8 +1158,7 @@ oop Reflection::invoke_method(oop method_mirror, Handle receiver, objArrayHandle
   methodHandle method(THREAD, m);
 
   // Coroutine work steal support
-  Thread* t = THREAD;
-  WispPostStealHandleUpdateMark w(t, &method);
+  WispPostStealHandleUpdateMark w((Thread *&)THREAD, &method);
 
   return invoke(klass, method, receiver, override, ptypes, rtype, args, true, THREAD);
 }
@@ -1188,8 +1186,7 @@ oop Reflection::invoke_constructor(oop constructor_mirror, objArrayHandle args,
   Handle receiver = klass->allocate_instance_handle(CHECK_NULL);
 
   // Coroutine work steal support
-  Thread* t = THREAD;
-  WispPostStealHandleUpdateMark w(t, &method);
+  WispPostStealHandleUpdateMark w((Thread *&)THREAD, &method);
 
   // Ignore result from call and return receiver
   invoke(klass, method, receiver, override, ptypes, T_VOID, args, false, CHECK_NULL);
diff --git a/src/hotspot/share/runtime/sharedRuntime.cpp b/src/hotspot/share/runtime/sharedRuntime.cpp
index c609321ac94..d8c3cdc3a26 100644
--- a/src/hotspot/share/runtime/sharedRuntime.cpp
+++ b/src/hotspot/share/runtime/sharedRuntime.cpp
@@ -558,6 +558,12 @@ address SharedRuntime::raw_exception_handler_for_return_address(JavaThread* curr
   return NULL;
 }
 
+#ifdef ASSERT
+JRT_LEAF(void, SharedRuntime::coroutine_switch_trace(JavaThread* current, void* arg))
+  tty->print_cr("Current thread = (%p, %s), arg = %p\n", current, current->name(), arg);
+JRT_END
+#endif
+
 
 JRT_LEAF(address, SharedRuntime::exception_handler_for_return_address(JavaThread* current, address return_address))
   return raw_exception_handler_for_return_address(current, return_address);
@@ -2151,8 +2157,7 @@ void SharedRuntime::monitor_enter_helper(oopDesc* obj, BasicLock* lock, JavaThre
   }
 
   // must place it befor EnableStealMark.
-  Thread* t = THREAD;
-  WispPostStealHandleUpdateMark w(current, t, __tiv);
+  WispPostStealHandleUpdateMark w(current, (Thread *&)THREAD, __tiv);
 
   // Coroutine work steal support
   EnableStealMark p(THREAD);
diff --git a/src/hotspot/share/runtime/sharedRuntime.hpp b/src/hotspot/share/runtime/sharedRuntime.hpp
index f2967a89a75..7c813c57cab 100644
--- a/src/hotspot/share/runtime/sharedRuntime.hpp
+++ b/src/hotspot/share/runtime/sharedRuntime.hpp
@@ -277,6 +277,11 @@ class SharedRuntime: AllStatic {
   static int dtrace_method_exit(JavaThread* thread, Method* m);
   static int wisp_yield(JavaThread* thread, Method* m);
 
+#ifdef ASSERT
+  // Trace wisp coroutine switch.
+  static void coroutine_switch_trace(JavaThread* thread, void* arg);
+#endif
+
   // Utility method for retrieving the Java thread id, returns 0 if the
   // thread is not a well formed Java thread.
   static jlong get_java_tid(Thread* thread);
diff --git a/src/hotspot/share/runtime/synchronizer.cpp b/src/hotspot/share/runtime/synchronizer.cpp
index c9bab5b59d2..d419001f55d 100644
--- a/src/hotspot/share/runtime/synchronizer.cpp
+++ b/src/hotspot/share/runtime/synchronizer.cpp
@@ -28,11 +28,13 @@
 #include "logging/log.hpp"
 #include "logging/logStream.hpp"
 #include "memory/allocation.inline.hpp"
+#include "memory/oopFactory.hpp"
 #include "memory/padded.hpp"
 #include "memory/resourceArea.hpp"
 #include "memory/universe.hpp"
 #include "oops/markWord.hpp"
 #include "oops/oop.inline.hpp"
+#include "oops/oopHandle.inline.hpp"
 #include "runtime/atomic.hpp"
 #include "runtime/biasedLocking.hpp"
 #include "runtime/coroutine.hpp"
@@ -432,7 +434,7 @@ void ObjectSynchronizer::exit(Handle object, BasicLock* lock, TRAPS) {
   markWord mark = object->mark();
   // We cannot check for Biased Locking if we are racing an inflation.
   assert(mark == markWord::INFLATING() ||
-         !mark->has_bias_pattern(), "should not see bias pattern here");
+         !mark.has_bias_pattern(), "should not see bias pattern here");
 
   markWord dhw = lock->displaced_header();
   if (dhw.value() == 0) {
@@ -443,10 +445,10 @@ void ObjectSynchronizer::exit(Handle object, BasicLock* lock, TRAPS) {
       // Only do diagnostics if we are not racing an inflation. Simply
       // exiting a recursive enter of a Java Monitor that is being
       // inflated is safe; see the has_monitor() comment below.
-      assert(!mark->is_neutral(), "invariant");
-      assert(!mark->has_locker() ||
-             THREAD->is_lock_owned((address)mark->locker()), "invariant");
-      if (mark->has_monitor()) {
+      assert(!mark.is_neutral(), "invariant");
+      assert(!mark.has_locker() ||
+             THREAD->is_lock_owned((address)mark.locker()), "invariant");
+      if (mark.has_monitor()) {
         // The BasicLock's displaced_header is marked as a recursive
         // enter and we have an inflated Java Monitor (ObjectMonitor).
         // This is a special case where the Java Monitor was inflated
@@ -455,7 +457,7 @@ void ObjectSynchronizer::exit(Handle object, BasicLock* lock, TRAPS) {
         // Monitor owner's stack and update the BasicLocks because a
         // Java Monitor can be asynchronously inflated by a thread that
         // does not own the Java Monitor.
-        ObjectMonitor * m = mark->monitor();
+        ObjectMonitor * m = mark.monitor();
         assert(((oop)(m->object()))->mark() == mark, "invariant");
         assert(m->is_entered(THREAD), "invariant");
       }
@@ -467,7 +469,7 @@ void ObjectSynchronizer::exit(Handle object, BasicLock* lock, TRAPS) {
   if (mark == markWord::from_pointer(lock)) {
     // If the object is stack-locked by the current thread, try to
     // swing the displaced header from the BasicLock back to the mark.
-    assert(dhw->is_neutral(), "invariant");
+    assert(dhw.is_neutral(), "invariant");
     if (object->cas_set_mark(dhw, mark) == mark) {
       return;
     }
@@ -666,7 +668,7 @@ void ObjectSynchronizer::jni_exit(oop obj, TRAPS) {
   // If this thread has locked the object, exit the monitor. We
   // intentionally do not use CHECK on check_owner because we must exit the
   // monitor even if an exception was already pending.
-  if (monitor->check_owner(THREAD)) {
+  if (monitor->check_owner(current)) {
     monitor->exit(current);
   }
 }
@@ -1846,8 +1848,14 @@ void ObjectSynchronizer::log_in_use_monitor_details(outputStream* out) {
   out->flush();
 }
 
-void SystemDictObjMonitor::lock(BasicLock* lock, TRAPS) {
+void SystemDictObjMonitor::lock(BasicLock* lock, Thread* current) {
   assert(UseWispMonitor, "SystemDictObjMonitor if only for UseWispMonitor");
+  if (!current->is_Java_thread()) {
+    _monitor->lock();
+    return;
+  }
+
+  JavaThread* jt = current->as_Java_thread();
   if (!is_obj_lock()) {
     _monitor->lock();
     if (is_obj_lock()) {
@@ -1855,48 +1863,70 @@ void SystemDictObjMonitor::lock(BasicLock* lock, TRAPS) {
       // now _monitor is Pointless. re-fetch the objectMonitor to
       // satisfy the critical section semantics.
       _monitor->unlock();
-      ObjectSynchronizer::enter(Handle(THREAD, _obj), lock, THREAD);
+      ObjectSynchronizer::enter(Handle(jt, _obj.resolve()), lock, jt);
     }
   } else {
-    ObjectSynchronizer::enter(Handle(THREAD, _obj), lock, THREAD);
+    ObjectSynchronizer::enter(Handle(jt, _obj.resolve()), lock, jt);
   }
 }
 
-void SystemDictObjMonitor::unlock(BasicLock* lock, TRAPS) {
+void SystemDictObjMonitor::unlock(BasicLock* lock, Thread* current) {
   assert(UseWispMonitor, "SystemDictObjMonitor if only for UseWispMonitor");
+  if (!current->is_Java_thread()) {
+    _monitor->unlock();
+    return;
+  }
+
+  JavaThread* jt = current->as_Java_thread();
   if (!is_obj_lock()) {
     _monitor->unlock();
   } else {
-    ObjectSynchronizer::exit(_obj, lock, THREAD);
+    ObjectSynchronizer::exit(_obj.resolve(), lock, jt);
   }
 }
 
-void SystemDictObjMonitor::wait(BasicLock* lock, TRAPS) {
+void SystemDictObjMonitor::wait(BasicLock* lock, Thread* current) {
   assert(UseWispMonitor, "SystemDictObjMonitor if only for UseWispMonitor");
+  assert(current->is_Java_thread(), "SystemDictObjMonitor::wait is only for JavaThread");
+
+  JavaThread* jt = current->as_Java_thread();
   if (!is_obj_lock()) {
     _monitor->wait();
     if (is_obj_lock()) {
       _monitor->unlock();
-      ObjectSynchronizer::enter(Handle(THREAD, _obj), lock, THREAD);
+      ObjectSynchronizer::enter(Handle(jt, _obj.resolve()), lock, jt);
     }
   } else {
-    ObjectSynchronizer::wait(Handle(THREAD, _obj), 0, THREAD);
+    ObjectSynchronizer::wait(Handle(jt, _obj.resolve()), 0, jt);
   }
 }
 
-void SystemDictObjMonitor::notify_all(TRAPS) {
+void SystemDictObjMonitor::notify_all(Thread* current) {
   assert(UseWispMonitor, "SystemDictObjMonitor if only for UseWispMonitor");
+  assert(current->is_Java_thread(), "SystemDictObjMonitor::notify_all is only for JavaThread");
+
+  JavaThread* jt = current->as_Java_thread();
   if (!is_obj_lock()) {
     _monitor->notify_all();
   } else {
-    ObjectSynchronizer::notifyall(Handle(THREAD, _obj), THREAD);
+    ObjectSynchronizer::notifyall(Handle(jt, _obj.resolve()), jt);
   }
 }
 
-void SystemDictObjMonitor::set_obj_lock(oop obj, TRAPS) {
-  MutexLocker mu(THREAD, _monitor);
+void SystemDictObjMonitor::create_obj_lock(Thread* current) {
+  guarantee(current->is_Java_thread(), "sanity check");
   assert(UseWispMonitor, "UseWispMonitor if off");
-  assert(_obj == NULL, "_obj already been set");
-  _obj = obj;
+
+  oop obj = oopFactory::new_intArray(0, current->as_Java_thread());
+  MutexLocker mu(current, _monitor);
+  _obj = OopHandle(Universe::vm_global(), obj);
   _monitor->notify_all();
 }
+
+oop SystemDictObjMonitor::obj() const {
+  return _obj.resolve();
+}
+
+bool SystemDictObjMonitor::is_obj_lock() const {
+  return _obj.peek() != NULL;
+}
diff --git a/src/hotspot/share/runtime/synchronizer.hpp b/src/hotspot/share/runtime/synchronizer.hpp
index fcaa31cb628..76e682d750f 100644
--- a/src/hotspot/share/runtime/synchronizer.hpp
+++ b/src/hotspot/share/runtime/synchronizer.hpp
@@ -27,6 +27,7 @@
 
 #include "memory/padded.hpp"
 #include "oops/markWord.hpp"
+#include "oops/oopHandle.hpp"
 #include "runtime/basicLock.hpp"
 #include "runtime/handles.hpp"
 #include "runtime/mutex.hpp"
@@ -213,32 +214,41 @@ class SystemDictMonitor : public CHeapObj {
     Monitor*  _monitor;
   public:
     SystemDictMonitor(Monitor* monitor): _monitor(monitor) { }
-    virtual void lock(BasicLock* lock, TRAPS)    { _monitor->lock();   }
-    virtual void unlock(BasicLock* lock, TRAPS)  { _monitor->unlock(); }
-    virtual void wait(BasicLock* lock, TRAPS)    { _monitor->wait();   }
-    virtual void notify_all(TRAPS)               { _monitor->notify_all(); }
-    virtual void set_obj_lock(oop obj, TRAPS) { ShouldNotReachHere(); }
+    virtual void lock(BasicLock* lock, Thread* current)    {
+        _monitor->lock();
+    }
+    virtual void unlock(BasicLock* lock, Thread* current)  {
+        _monitor->unlock();
+    }
+    virtual void wait(BasicLock* lock, Thread* current)    {
+        _monitor->wait();
+    }
+    virtual void notify_all(Thread* current)               {
+        _monitor->notify_all();
+    }
+    virtual void create_obj_lock(Thread* current) {
+		ShouldNotReachHere();
+    }
     Monitor* monitor()         const    { return _monitor;               }
     virtual oop obj()          const    { return NULL;                   }
     virtual bool is_obj_lock() const    { return false;                  }
-    virtual void oops_do(OopClosure* f) { }
 };
 
 class SystemDictObjMonitor : public SystemDictMonitor {
   private:
-    oop       _obj;
+    // Prefer to use obj lock. Only use monitor when obj lock is not set.
+    OopHandle       _obj;
   public:
-    SystemDictObjMonitor(Monitor* monitor): SystemDictMonitor(monitor), _obj(NULL) {
+    SystemDictObjMonitor(Monitor* monitor) : SystemDictMonitor(monitor) {
       assert(UseWispMonitor, "SystemDictObjMonitor if only for UseWispMonitor");
     }
-    virtual void lock(BasicLock* lock, TRAPS);
-    virtual void unlock(BasicLock* lock, TRAPS);
-    virtual void wait(BasicLock* lock, TRAPS);
-    virtual void notify_all(TRAPS);
-    virtual void set_obj_lock(oop obj, TRAPS);
-    virtual oop obj()          const    { return _obj;                   }
-    virtual bool is_obj_lock() const    { return _obj != NULL;           }
-    virtual void oops_do(OopClosure* f) { if (_obj != NULL)  f->do_oop(&_obj);   }
+    virtual void lock(BasicLock* lock, Thread* current);
+    virtual void unlock(BasicLock* lock, Thread* current);
+    virtual void wait(BasicLock* lock, Thread* current);
+    virtual void notify_all(Thread* current);
+    virtual void create_obj_lock(Thread* current);
+    virtual oop obj() const;
+    virtual bool is_obj_lock() const;
 };
 
 #endif // SHARE_RUNTIME_SYNCHRONIZER_HPP
diff --git a/src/hotspot/share/runtime/thread.cpp b/src/hotspot/share/runtime/thread.cpp
index a04fbeb8d6c..9ae42e1a587 100644
--- a/src/hotspot/share/runtime/thread.cpp
+++ b/src/hotspot/share/runtime/thread.cpp
@@ -82,6 +82,7 @@
 #include "runtime/flags/jvmFlagLimit.hpp"
 #include "runtime/deoptimization.hpp"
 #include "runtime/frame.inline.hpp"
+#include "runtime/globals.hpp"
 #include "runtime/handles.inline.hpp"
 #include "runtime/handshake.hpp"
 #include "runtime/init.hpp"
@@ -310,7 +311,7 @@ Thread::Thread() {
     // and that happens just before Thread::current is set. No other thread
     // can attach as the VM is not created yet, so they can't execute this code.
     // If the main thread creates other threads before the barrier set that is an error.
-    assert(Thread::current_or_null() == NULL, "creating thread before barrier set");
+    assert(UseWispMonitor || Thread::current_or_null() == NULL, "creating thread before barrier set");
   }
 
   MACOS_AARCH64_ONLY(DEBUG_ONLY(_wx_init = false));
@@ -1134,9 +1135,7 @@ JavaThread::JavaThread() :
   // Setup safepoint state info for this thread
   ThreadSafepointState::create(this);
 
-#ifdef ASSERT
   _java_call_counter = 0;
-#endif
 
   SafepointMechanism::initialize_header(this);
 
@@ -1175,10 +1174,7 @@ void JavaThread::interrupt() {
 }
 
 bool JavaThread::is_interrupted(bool clear_interrupted) {
-  JavaThread* thread = JavaThread::current();
-  if (UseWispMonitor && thread->is_Wisp_thread()) {
-    thread = ((WispThread*) thread)->thread();
-  }
+  guarantee(!UseWispMonitor || !is_Wisp_thread(), "sanity check");
   debug_only(check_for_dangling_thread_pointer(this);)
 
   if (_threadObj.peek() == NULL) {
@@ -1216,6 +1212,22 @@ bool JavaThread::is_interrupted(bool clear_interrupted) {
   return interrupted;
 }
 
+bool JavaThread::clear_interrupt_for_wisp() {
+  assert(EnableCoroutine, "Coroutine is disabled");
+  // If we only use -XX:+EnableCoroutine and
+  // -Dcom.alibaba.transparentAsync=true, we will fall here, so we cannot use
+  // `assert(UseWispMonitor)` only.
+  JavaThread* thread = this;
+  if (UseWispMonitor && thread->is_Wisp_thread()) {
+    thread = ((WispThread*)thread)->thread();
+  }
+
+  bool interrupted = thread->is_interrupted(false);
+  java_lang_Thread::set_interrupted(thread->threadObj(), false);
+
+  return interrupted;
+}
+
 void JavaThread::block_if_vm_exited() {
   if (_terminated == _vm_exited) {
     // _vm_exited is set at safepoint, and Threads_lock is never released
@@ -1525,7 +1537,7 @@ void JavaThread::exit(bool destroy_vm, ExitType exit_type) {
       // SurrogateLockerThread and ServiceThread are "is_hidden_from_external_view()"
       !is_jvmti_agent_thread()) {
     assert(!UseWispMonitor || destroy_vm ||
-        java_lang_Thread::park_event(_threadObj), "park_event should been set");
+        java_lang_Thread::park_event(threadObj()), "park_event should been set");
     EXCEPTION_MARK;
     JavaValue result(T_VOID);
     JavaCalls::call_virtual(&result,
diff --git a/src/hotspot/share/runtime/thread.hpp b/src/hotspot/share/runtime/thread.hpp
index 543558a5ce2..697d6ce4d1b 100644
--- a/src/hotspot/share/runtime/thread.hpp
+++ b/src/hotspot/share/runtime/thread.hpp
@@ -1505,9 +1505,6 @@ class JavaThread: public Thread {
   void thread_main_inner();
   virtual void post_run();
 
- public:
-//  static ByteSize privileged_stack_top_offset()         { return byte_offset_of(JavaThread, _privileged_stack_top); }
-
  public:
   // Thread local information maintained by JVMTI.
   void set_jvmti_thread_state(JvmtiThreadState *value)                           { _jvmti_thread_state = value; }
@@ -1654,6 +1651,7 @@ class JavaThread: public Thread {
   // java.lang.Thread interruption support
   void interrupt();
   bool is_interrupted(bool clear_interrupted);
+  bool clear_interrupt_for_wisp();
 
   static OopStorage* thread_oop_storage();
 
diff --git a/src/hotspot/share/runtime/vframe.hpp b/src/hotspot/share/runtime/vframe.hpp
index bf9473ff2d7..3b09bc4b0ab 100644
--- a/src/hotspot/share/runtime/vframe.hpp
+++ b/src/hotspot/share/runtime/vframe.hpp
@@ -340,6 +340,7 @@ class vframeStream : public vframeStreamCommon {
   vframeStream(JavaThread* thread, bool stop_at_java_call_stub = false, bool process_frames = true);
 
   Thread *& thread_ref()    {
+    assert(EnableCoroutine, "EnableCoroutine is off");
     return (Thread *&)_thread;
   }
 
diff --git a/src/hotspot/share/utilities/exceptions.cpp b/src/hotspot/share/utilities/exceptions.cpp
index 3f6ef1deff4..84352322473 100644
--- a/src/hotspot/share/utilities/exceptions.cpp
+++ b/src/hotspot/share/utilities/exceptions.cpp
@@ -291,11 +291,6 @@ Handle Exceptions::new_exception(JavaThread* thread, Symbol* name,
                                 signature,
                                 args,
                                 thread);
-
-    {
-      guarantee(!EnableSteal || thread == Thread::current(), "fatal: stealed");
-    }
-
   }
 
   // Check if another exception was thrown in the process, if so rethrow that one
diff --git a/src/java.base/share/classes/java/dyn/CoroutineBase.java b/src/java.base/share/classes/java/dyn/CoroutineBase.java
index 7ee5f4facad..ce7f92ca597 100644
--- a/src/java.base/share/classes/java/dyn/CoroutineBase.java
+++ b/src/java.base/share/classes/java/dyn/CoroutineBase.java
@@ -28,7 +28,7 @@
 import jdk.internal.access.SharedSecrets;
 
 /**
- * * Abstract of coroutines.
+ * Abstract of coroutines.
  */
 public abstract class CoroutineBase {
     transient long data;
diff --git a/src/java.base/share/classes/java/lang/Thread.java b/src/java.base/share/classes/java/lang/Thread.java
index bb22af09a8d..a2879848877 100644
--- a/src/java.base/share/classes/java/lang/Thread.java
+++ b/src/java.base/share/classes/java/lang/Thread.java
@@ -1099,10 +1099,10 @@ public void interrupt() {
             synchronized (blockerLock) {
                 Interruptible b = blocker;
                 if (b != null) {
-                    interrupted = true;
                     if (WEA != null && wispTask != null) {
                         WEA.interrupt(wispTask);
                     } else {
+                        interrupted = true;
                         interrupt0();           // Just to set the interrupt flag
                     }
                     b.interrupt(this);
@@ -1110,10 +1110,10 @@ public void interrupt() {
                 }
             }
         }
-        interrupted = true;
         if (WEA != null && wispTask != null) {
             WEA.interrupt(wispTask);
         } else {
+            interrupted = true;
             interrupt0();
         }
     }
diff --git a/src/java.base/share/classes/java/net/DatagramSocket.java b/src/java.base/share/classes/java/net/DatagramSocket.java
index 020b2e1d69b..91262636b46 100644
--- a/src/java.base/share/classes/java/net/DatagramSocket.java
+++ b/src/java.base/share/classes/java/net/DatagramSocket.java
@@ -267,6 +267,10 @@ DatagramSocket delegate() {
      * @param delegate The wrapped DatagramSocket implementation, or null.
      */
     DatagramSocket(DatagramSocket delegate) {
+        if (WispEngine.transparentWispSwitch()) {
+            asyncImpl = new WispUdpSocketImpl(this);
+            assert delegate == null;
+        }
         assert delegate == null
                 || delegate instanceof NetMulticastSocket
                 || delegate instanceof sun.nio.ch.DatagramSocketAdaptor;
@@ -332,12 +336,15 @@ protected DatagramSocket(DatagramSocketImpl impl) {
      * @since   1.4
      */
     public DatagramSocket(SocketAddress bindaddr) throws SocketException {
-        // if (WispEngine.transparentWispSwitch()) {
-        //     asyncImpl = new WispUdpSocketImpl(this);
-        // } else {
-        //     DatagramSocket(createDelegate(bindaddr, DatagramSocket.class));
-        // }
         this(createDelegate(bindaddr, DatagramSocket.class));
+        if (WispEngine.transparentWispSwitch() && bindaddr != null) {
+            try {
+                bind(bindaddr);
+            } finally {
+                if (!isBound())
+                    close();
+            }
+        }
     }
 
     /**
@@ -1489,6 +1496,9 @@ private static SocketException toSocketException(IOException e) {
      */
     static  T createDelegate(SocketAddress bindaddr, Class type)
             throws SocketException {
+        if (WispEngine.transparentWispSwitch()) {
+            return null;
+        }
 
         // Temporary solution until JDK-8237352 is addressed
         if (bindaddr == NO_DELEGATE) return null;
diff --git a/src/java.base/share/classes/sun/nio/ch/ServerSocketChannelImpl.java b/src/java.base/share/classes/sun/nio/ch/ServerSocketChannelImpl.java
index 4aeadee8d45..9572948dc80 100644
--- a/src/java.base/share/classes/sun/nio/ch/ServerSocketChannelImpl.java
+++ b/src/java.base/share/classes/sun/nio/ch/ServerSocketChannelImpl.java
@@ -394,14 +394,16 @@ public SocketChannel accept() throws IOException {
             final boolean wispAndBlocking = WispEngine.transparentWispSwitch() && blocking;
             try {
                 begin(blocking);
+                if (wispAndBlocking) {
+                    IOUtil.configureBlocking(fd, false);
+                }
                 n = implAccept(this.fd, newfd, saa);
                 if (blocking) {
                     while (IOStatus.okayToRetry(n) && isOpen()) {
                         if (wispAndBlocking && n < 0) {
                             WEA.registerEvent(this, SelectionKey.OP_ACCEPT);
                             WEA.park(-1);
-                        }
-                        else {
+                        } else {
                             park(Net.POLLIN);
                         }
                         n = implAccept(this.fd, newfd, saa);
diff --git a/test/hotspot/jtreg/runtime/coroutine/C1ThrowSyncExceptionTest.java b/test/hotspot/jtreg/runtime/coroutine/C1ThrowSyncExceptionTest.java
index 686fc28ee28..53532d5aa26 100644
--- a/test/hotspot/jtreg/runtime/coroutine/C1ThrowSyncExceptionTest.java
+++ b/test/hotspot/jtreg/runtime/coroutine/C1ThrowSyncExceptionTest.java
@@ -1,7 +1,7 @@
 /*
  * @test
  * @summary test a special wisp unpark case for C1 compiled method
- * @modules java.base/jdk.internal.misc
+ * @modules java.base/jdk.internal.access
  * @run main/othervm  -XX:-UseBiasedLocking -XX:+EnableCoroutine -XX:+UseWispMonitor -Dcom.alibaba.transparentAsync=true -XX:TieredStopAtLevel=1 C1ThrowSyncExceptionTest
  */
 
diff --git a/test/hotspot/jtreg/runtime/coroutine/TestAvoidDeoptCoroutineMethod.java b/test/hotspot/jtreg/runtime/coroutine/TestAvoidDeoptCoroutineMethod.java
index 5d01c2ff442..edcb267c6c6 100644
--- a/test/hotspot/jtreg/runtime/coroutine/TestAvoidDeoptCoroutineMethod.java
+++ b/test/hotspot/jtreg/runtime/coroutine/TestAvoidDeoptCoroutineMethod.java
@@ -2,7 +2,7 @@
  * @test TestAvoidDeoptCoroutineMethod
  * @library /test/lib
  * @build sun.hotspot.WhiteBox
- * @run driver ClassFileInstaller sun.hotspot.WhiteBox
+ * @run driver jdk.test.lib.helpers.ClassFileInstaller sun.hotspot.WhiteBox
  * @build TestAvoidDeoptCoroutineMethod
  * @run main/othervm -XX:+EnableCoroutine -Xmx10m -Xms10m -Xbootclasspath/a:. -XX:+UnlockDiagnosticVMOptions -XX:+WhiteBoxAPI TestAvoidDeoptCoroutineMethod
  * @summary test avoid coroutine intrinsic method to be deoptimized
diff --git a/test/hotspot/jtreg/runtime/coroutine/TimeSliceSyncTest.java b/test/hotspot/jtreg/runtime/coroutine/TimeSliceSyncTest.java
index cf87fcc9257..151ba4d8caa 100644
--- a/test/hotspot/jtreg/runtime/coroutine/TimeSliceSyncTest.java
+++ b/test/hotspot/jtreg/runtime/coroutine/TimeSliceSyncTest.java
@@ -2,7 +2,7 @@
  * @test
  * @library /testlibrary
  * @summary This test ensures that coroutine time slice feature won't cause hang.
- * @modules java.base/jdk.internal.misc
+ * @modules java.base/jdk.internal.access
  * @run main/othervm  -XX:-UseBiasedLocking -XX:+EnableCoroutine -XX:+UseWispMonitor -Dcom.alibaba.wisp.transparentWispSwitch=true -XX:-Inline -XX:+EnableCoroutineTimeSlice TimeSliceSyncTest
  * @run main/othervm  -XX:-UseBiasedLocking -XX:+EnableCoroutine -XX:+UseWispMonitor -Dcom.alibaba.wisp.transparentWispSwitch=true -XX:-Inline -XX:+EnableCoroutineTimeSlice -Dcom.alibaba.wisp.version=2 TimeSliceSyncTest
  */
diff --git a/test/hotspot/jtreg/runtime/coroutine/Wisp2SwitchTest.java b/test/hotspot/jtreg/runtime/coroutine/Wisp2SwitchTest.java
index 6f7a9c0c951..777041d8d01 100644
--- a/test/hotspot/jtreg/runtime/coroutine/Wisp2SwitchTest.java
+++ b/test/hotspot/jtreg/runtime/coroutine/Wisp2SwitchTest.java
@@ -2,7 +2,7 @@
  * @test
  * @summary test wisp2 switch
  * @library /test/lib
- * @modules java.base/jdk.internal.misc
+ * @modules java.base/jdk.internal.access
  * @modules java.base/com.alibaba.wisp.engine:+open
  * @run main/othervm -XX:+UseWisp2  Wisp2SwitchTest
  */
diff --git a/test/hotspot/jtreg/runtime/coroutine/Wisp2SwitchTest2.java b/test/hotspot/jtreg/runtime/coroutine/Wisp2SwitchTest2.java
index 7baab36c447..11edee4a3aa 100644
--- a/test/hotspot/jtreg/runtime/coroutine/Wisp2SwitchTest2.java
+++ b/test/hotspot/jtreg/runtime/coroutine/Wisp2SwitchTest2.java
@@ -2,7 +2,7 @@
  * @test
  * @summary test XX:+UseWisp2 switch with -Dcom.alibaba.wisp.allThreadAsWisp=false
  * @library /test/lib
- * @modules java.base/jdk.internal.misc
+ * @modules java.base/jdk.internal.access
  * @modules java.base/com.alibaba.wisp.engine:+open
  * @run main/othervm -XX:+UseWisp2 -Dcom.alibaba.wisp.allThreadAsWisp=false Wisp2SwitchTest2
  */
diff --git a/test/hotspot/jtreg/runtime/coroutine/jniDetachThreadHoldingMonitorTest.sh b/test/hotspot/jtreg/runtime/coroutine/jniDetachThreadHoldingMonitorTest.sh
deleted file mode 100644
index 0f266fcc012..00000000000
--- a/test/hotspot/jtreg/runtime/coroutine/jniDetachThreadHoldingMonitorTest.sh
+++ /dev/null
@@ -1,18 +0,0 @@
-#!/bin/sh
-
-## @test
-##
-## @summary test DetachCurrentThread unpark
-## @run shell jniDetachThreadHoldingMonitorTest.sh
-##
-
-
-export LD_LIBRARY_PATH=.:${COMPILEJAVA}/lib/server:/usr/lib:$LD_LIBRARY_PATH
-
-g++ -DLINUX -o jniDetachThreadHoldingMonitorTest \
-    -I${COMPILEJAVA}/include -I${COMPILEJAVA}/include/linux \
-    -L${COMPILEJAVA}/lib/server \
-    -ljvm -lpthread ${TESTSRC}/jniDetachThreadHoldingMonitorTest.c
-
-./jniDetachThreadHoldingMonitorTest
-exit $?
diff --git a/test/hotspot/jtreg/runtime/coroutine/jniMonitorExitTest.sh b/test/hotspot/jtreg/runtime/coroutine/jniMonitorExitTest.sh
deleted file mode 100644
index ede41f4360c..00000000000
--- a/test/hotspot/jtreg/runtime/coroutine/jniMonitorExitTest.sh
+++ /dev/null
@@ -1,19 +0,0 @@
-#!/bin/sh
-
-## @test
-##
-## @summary test jni MonitorExit
-## @run shell jniMonitorExitTest.sh
-##
-
-
-export LD_LIBRARY_PATH=.:${COMPILEJAVA}/lib/server:/usr/lib:$LD_LIBRARY_PATH
-echo ${COMPILEJAVA}
-echo $LD_LIBRARY_PATH
-g++ -DLINUX -o jniMonitorExitTest \
-    -I${COMPILEJAVA}/include -I${COMPILEJAVA}/include/linux \
-    -L${COMPILEJAVA}/lib/server \
-    -ljvm -lpthread ${TESTSRC}/jniMonitorExitTest.c
-
-./jniMonitorExitTest
-exit $?
diff --git a/test/hotspot/jtreg/runtime/jni/registerNativesWarning/TestRegisterNativesWarning.java b/test/hotspot/jtreg/runtime/jni/registerNativesWarning/TestRegisterNativesWarning.java
index 77aabcc7a01..cd7bce11b25 100644
--- a/test/hotspot/jtreg/runtime/jni/registerNativesWarning/TestRegisterNativesWarning.java
+++ b/test/hotspot/jtreg/runtime/jni/registerNativesWarning/TestRegisterNativesWarning.java
@@ -43,9 +43,12 @@ public class TestRegisterNativesWarning {
 
     /*
      * We will replace:
-     *   java/lang/Thread.java:    public static native void yield();
+     *   java/lang/System.java:    public static native void currentTimeMillis();
      *
      * as it is simple and innocuous.
+     *
+     * KBOX: yield() in Thread.java is not a native method after we
+     * port wisp coroutine to JDK. Thus we use currentTimeMillis() instead.
      */
     native static void test(Class jlThread);
 
@@ -55,13 +58,13 @@ static class Tester {
         public static void main(String[] args) throws Exception {
             System.out.println("Running test() in class loader " +
                                Tester.class.getClassLoader());
-            test(Thread.class);
-            Thread.yield();
+            test(System.class);
+            System.currentTimeMillis();
         }
     }
 
     public static void main(String[] args) throws Exception {
-        String warning = "Re-registering of platform native method: java.lang.Thread.yield()V from code in a different classloader";
+        String warning = "Re-registering of platform native method: java.lang.System.currentTimeMillis()J from code in a different classloader";
 
         String cp = Utils.TEST_CLASS_PATH;
         String libp = Utils.TEST_NATIVE_PATH;
diff --git a/test/hotspot/jtreg/runtime/jni/registerNativesWarning/libregisterNativesWarning.c b/test/hotspot/jtreg/runtime/jni/registerNativesWarning/libregisterNativesWarning.c
index 1ced153f553..692492c7050 100644
--- a/test/hotspot/jtreg/runtime/jni/registerNativesWarning/libregisterNativesWarning.c
+++ b/test/hotspot/jtreg/runtime/jni/registerNativesWarning/libregisterNativesWarning.c
@@ -30,6 +30,9 @@
  *   java/lang/Thread.java:    public static native void yield();
  *
  * as it is simple and innocuous.
+ *
+ * KBOX: yield() in Thread.java is not a native method after we
+ * port wisp coroutine to JDK. Thus we use currentTimeMillis() instead.
  */
 static void myYield(JNIEnv* env, jclass cls) {
   printf("myYield executed\n");
@@ -41,8 +44,8 @@ Java_TestRegisterNativesWarning_test
 (JNIEnv *env, jclass cls, jclass jlThread) {
   JNINativeMethod nativeMethods[] = {
     {
-      (char*) "yield",  // name
-      (char*) "()V",    // sig
+      (char*) "currentTimeMillis",  // name
+      (char*) "()J",    // sig
       (void*) myYield   // native method ptr
     }
   };
diff --git a/test/hotspot/jtreg/vmTestbase/nsk/monitoring/share/ThreadController.java b/test/hotspot/jtreg/vmTestbase/nsk/monitoring/share/ThreadController.java
index a6c13bcee8a..c399084f3c7 100644
--- a/test/hotspot/jtreg/vmTestbase/nsk/monitoring/share/ThreadController.java
+++ b/test/hotspot/jtreg/vmTestbase/nsk/monitoring/share/ThreadController.java
@@ -655,6 +655,7 @@ public SleepingThread(ThreadController controller, String name, Log log, Threads
         expectedLength += 3;
 
         expectedMethods.add(Thread.class.getName() + ".sleep");
+        expectedMethods.add(Thread.class.getName() + ".sleep0");
         expectedMethods.add(SleepingThread.class.getName() + ".run");
 
         switch (controller.invocationType) {
@@ -722,6 +723,7 @@ public RunningThread(ThreadController controller, String name, Log log, ThreadsG
         expectedLength += 2;
 
         expectedMethods.add(Thread.class.getName() + ".yield");
+        expectedMethods.add(Thread.class.getName() + ".yield0");
 
         switch (controller.invocationType) {
             case ThreadController.JAVA_TYPE:
diff --git a/test/hotspot/jtreg/vmTestbase/nsk/monitoring/share/thread/RunningThread.java b/test/hotspot/jtreg/vmTestbase/nsk/monitoring/share/thread/RunningThread.java
index 7f1db6d9325..25da293c5ec 100644
--- a/test/hotspot/jtreg/vmTestbase/nsk/monitoring/share/thread/RunningThread.java
+++ b/test/hotspot/jtreg/vmTestbase/nsk/monitoring/share/thread/RunningThread.java
@@ -36,7 +36,8 @@ public class RunningThread extends RecursiveMonitoringThread {
         private Object readyLock = new Object();
         private static final String[] expectedMethods = {
                 "nsk.monitoring.share.thread.RunningThread.runInside",
-                "java.lang.Thread.yield"
+                "java.lang.Thread.yield",
+                "java.lang.Thread.yield0"
         };
 
         public RunningThread(Log log, RunType recursionType, int maxDepth) {
diff --git a/test/hotspot/jtreg/vmTestbase/nsk/monitoring/share/thread/SleepingThread.java b/test/hotspot/jtreg/vmTestbase/nsk/monitoring/share/thread/SleepingThread.java
index 05e3b9de418..5f97d0529da 100644
--- a/test/hotspot/jtreg/vmTestbase/nsk/monitoring/share/thread/SleepingThread.java
+++ b/test/hotspot/jtreg/vmTestbase/nsk/monitoring/share/thread/SleepingThread.java
@@ -36,6 +36,7 @@ public class SleepingThread extends RecursiveMonitoringThread {
         private Object readyLock = new Object();
         private static final String[] expectedMethods = {
                 "java.lang.Thread.sleep",
+                "java.lang.Thread.sleep0",
                 "nsk.monitoring.share.thread.SleepingThread.runInside"
         };
 
diff --git a/test/hotspot/jtreg/vmTestbase/nsk/monitoring/stress/thread/strace001.java b/test/hotspot/jtreg/vmTestbase/nsk/monitoring/stress/thread/strace001.java
index 77b881ed0cf..8e620ce5db0 100644
--- a/test/hotspot/jtreg/vmTestbase/nsk/monitoring/stress/thread/strace001.java
+++ b/test/hotspot/jtreg/vmTestbase/nsk/monitoring/stress/thread/strace001.java
@@ -140,7 +140,9 @@ private static boolean fillTrace() {
         switch (controller.getInvocationType()) {
             case ThreadController.JAVA_TYPE:
                 expectedTrace = new String[] {
-                    "java.lang.Thread.sleep"
+                    "java.lang.Thread.sleep0"
+                    , "java.lang.Thread.sleep"
+                    , "java.lang.Thread.yield0"
                     , "java.lang.Thread.yield"
                     , THREAD_NAME + ".waitForSign"
                     , THREAD_NAME + ".recursionJava"
@@ -150,7 +152,9 @@ private static boolean fillTrace() {
 
             case ThreadController.NATIVE_TYPE:
                 expectedTrace = new String[] {
-                    "java.lang.Thread.sleep"
+                    "java.lang.Thread.sleep0"
+                    , "java.lang.Thread.sleep"
+                    , "java.lang.Thread.yield0"
                     , "java.lang.Thread.yield"
                     , THREAD_NAME + ".waitForSign"
                     , THREAD_NAME + ".recursionNative"
@@ -160,7 +164,9 @@ private static boolean fillTrace() {
 
             case ThreadController.MIXED_TYPE:
                 expectedTrace = new String[] {
-                    "java.lang.Thread.sleep"
+                    "java.lang.Thread.sleep0"
+                    , "java.lang.Thread.sleep"
+                    , "java.lang.Thread.yield0"
                     , "java.lang.Thread.yield"
                     , THREAD_NAME + ".waitForSign"
                     , THREAD_NAME + ".recursionNative"
@@ -194,7 +200,7 @@ private static void printStackTrace(StackTraceElement[] elements) {
     // The method performs checks of the stack trace
     private static boolean checkTrace(StackTraceElement[] elements) {
         int length = elements.length;
-        int expectedLength = depth +3;
+        int expectedLength = depth +4;
         boolean result = true;
 
         // Check the length of the trace. It must not be greater than
diff --git a/test/hotspot/jtreg/vmTestbase/nsk/share/locks/LockingThread.java b/test/hotspot/jtreg/vmTestbase/nsk/share/locks/LockingThread.java
index ab97a02fdf5..f80a43f5536 100644
--- a/test/hotspot/jtreg/vmTestbase/nsk/share/locks/LockingThread.java
+++ b/test/hotspot/jtreg/vmTestbase/nsk/share/locks/LockingThread.java
@@ -411,6 +411,17 @@ public void exitSingleFrame() {
         interrupt();
     }
 
+    // This function add one more stack frame when calling relinquishedMonitor.wait().
+    private void myWait(Object relinquishedMonitor) {
+        try {
+            relinquishedMonitor.wait(0);
+        } catch (Exception e) {
+            executedWithErrors = true;
+            log("Unexpected exception: " + e);
+            e.printStackTrace(log.getOutStream());
+        }
+    }
+
     // LockingThread call this method when required state is reached
     private void doWait() {
         while (true) {
@@ -440,7 +451,7 @@ private void doWait() {
                     // and this method waits when LockingThred change state to 'Thread.State.WAITING'
 
                     while (relinquishMonitor)
-                        relinquishedMonitor.wait(0);
+                        myWait(relinquishedMonitor);
 
                     log("Acquire relinquished monitor: " + relinquishedMonitor);
                 } catch (Exception e) {
@@ -488,7 +499,9 @@ static synchronized void synchronizedStaticMethod(LockingThread lockingThread) {
     int expectedDepth() {
         // for each monitor call 2 methods: createStackFrame() and method which acquire monitor
         // + when stack creation is finished call 3 methods: createStackFrame()->doWait()->sleep()
-        return (stackFramesDescription.size() - currentIndex) * 2 + 3;
+        // return (stackFramesDescription.size() - currentIndex) * 2 + 3;
+        // After port wisp coroutine, need to add 1 more call after sleep(), i.e. sleep()->sleep0()
+        return (stackFramesDescription.size() - currentIndex) * 2 + 4;
     }
 
     private native void nativeJNIMonitorEnter(Object object);
diff --git a/test/hotspot/jtreg/vmTestbase/nsk/stress/strace/strace001.java b/test/hotspot/jtreg/vmTestbase/nsk/stress/strace/strace001.java
index ec92fc3df47..5ff4a6d7ec2 100644
--- a/test/hotspot/jtreg/vmTestbase/nsk/stress/strace/strace001.java
+++ b/test/hotspot/jtreg/vmTestbase/nsk/stress/strace/strace001.java
@@ -71,6 +71,7 @@ public class strace001 {
             "java.lang.Object.wait",
             "java.lang.Thread.exit",
             "java.lang.Thread.yield",
+            "java.lang.Thread.yield0",
             "java.lang.ThreadGroup.remove",
             "java.lang.ThreadGroup.threadTerminated",
             "nsk.stress.strace.strace001Thread.run",
diff --git a/test/hotspot/jtreg/vmTestbase/nsk/stress/strace/strace002.java b/test/hotspot/jtreg/vmTestbase/nsk/stress/strace/strace002.java
index 793a2e680f4..2b16326491a 100644
--- a/test/hotspot/jtreg/vmTestbase/nsk/stress/strace/strace002.java
+++ b/test/hotspot/jtreg/vmTestbase/nsk/stress/strace/strace002.java
@@ -77,6 +77,7 @@ public class strace002 {
             "java.lang.Object.wait",
             "java.lang.Thread.exit",
             "java.lang.Thread.yield",
+            "java.lang.Thread.yield0",
             "java.lang.ThreadGroup.remove",
             "java.lang.ThreadGroup.threadTerminated",
             "nsk.stress.strace.strace002Thread.run",
diff --git a/test/hotspot/jtreg/vmTestbase/nsk/stress/strace/strace003.java b/test/hotspot/jtreg/vmTestbase/nsk/stress/strace/strace003.java
index f20bb4a5cf6..28a779a5203 100644
--- a/test/hotspot/jtreg/vmTestbase/nsk/stress/strace/strace003.java
+++ b/test/hotspot/jtreg/vmTestbase/nsk/stress/strace/strace003.java
@@ -117,6 +117,7 @@ public class strace003 {
             "java.lang.Object.wait",
             "java.lang.Thread.exit",
             "java.lang.Thread.yield",
+            "java.lang.Thread.yield0",
             "java.lang.ThreadGroup.remove",
             "java.lang.ThreadGroup.threadTerminated",
             "nsk.stress.strace.strace003Thread.run",
diff --git a/test/hotspot/jtreg/vmTestbase/nsk/stress/strace/strace004.java b/test/hotspot/jtreg/vmTestbase/nsk/stress/strace/strace004.java
index b28b5b119b4..667421c7a1e 100644
--- a/test/hotspot/jtreg/vmTestbase/nsk/stress/strace/strace004.java
+++ b/test/hotspot/jtreg/vmTestbase/nsk/stress/strace/strace004.java
@@ -110,6 +110,7 @@ public class strace004 {
             "java.lang.Object.wait",
             "java.lang.Thread.exit",
             "java.lang.Thread.yield",
+            "java.lang.Thread.yield0",
             "java.lang.ThreadGroup.remove",
             "java.lang.ThreadGroup.threadTerminated",
             "nsk.stress.strace.strace004Thread.run",
diff --git a/test/hotspot/jtreg/vmTestbase/nsk/stress/strace/strace005.java b/test/hotspot/jtreg/vmTestbase/nsk/stress/strace/strace005.java
index 6badd2bf1fb..40e9c380a06 100644
--- a/test/hotspot/jtreg/vmTestbase/nsk/stress/strace/strace005.java
+++ b/test/hotspot/jtreg/vmTestbase/nsk/stress/strace/strace005.java
@@ -136,6 +136,7 @@ public class strace005 {
             "java.lang.Object.wait",
             "java.lang.Thread.exit",
             "java.lang.Thread.yield",
+            "java.lang.Thread.yield0",
             "java.lang.ThreadGroup.remove",
             "java.lang.ThreadGroup.threadTerminated",
             "nsk.stress.strace.strace005Thread.run",
diff --git a/test/hotspot/jtreg/vmTestbase/nsk/stress/strace/strace006.java b/test/hotspot/jtreg/vmTestbase/nsk/stress/strace/strace006.java
index 20813247438..e9375727a45 100644
--- a/test/hotspot/jtreg/vmTestbase/nsk/stress/strace/strace006.java
+++ b/test/hotspot/jtreg/vmTestbase/nsk/stress/strace/strace006.java
@@ -76,6 +76,7 @@ public class strace006 {
             "java.lang.Object.wait",
             "java.lang.Thread.exit",
             "java.lang.Thread.yield",
+            "java.lang.Thread.yield0",
             "java.lang.ThreadGroup.remove",
             "java.lang.ThreadGroup.threadTerminated",
             "nsk.stress.strace.strace006Thread.run",
diff --git a/test/hotspot/jtreg/vmTestbase/nsk/stress/strace/strace007.java b/test/hotspot/jtreg/vmTestbase/nsk/stress/strace/strace007.java
index 65fe92cab89..74189b1f7ae 100644
--- a/test/hotspot/jtreg/vmTestbase/nsk/stress/strace/strace007.java
+++ b/test/hotspot/jtreg/vmTestbase/nsk/stress/strace/strace007.java
@@ -72,6 +72,7 @@ public class strace007 {
     static final int SLEEP_TIME = 50;
     static final String[] EXPECTED_METHODS = {
             "java.lang.Thread.sleep",
+            "java.lang.Thread.sleep0",
             "nsk.stress.strace.strace007Thread.run",
             "nsk.stress.strace.strace007Thread.recursiveMethod"
     };
diff --git a/test/hotspot/jtreg/vmTestbase/nsk/stress/strace/strace008.java b/test/hotspot/jtreg/vmTestbase/nsk/stress/strace/strace008.java
index c110af8b421..49a84f77ddf 100644
--- a/test/hotspot/jtreg/vmTestbase/nsk/stress/strace/strace008.java
+++ b/test/hotspot/jtreg/vmTestbase/nsk/stress/strace/strace008.java
@@ -76,6 +76,7 @@ public class strace008 {
     static final String NATIVE_LIB = "strace008";
     static final String[] EXPECTED_METHODS = {
             "java.lang.Thread.sleep",
+            "java.lang.Thread.sleep0",
             "nsk.stress.strace.strace008Thread.run",
             "nsk.stress.strace.strace008Thread.recursiveMethod"
     };
diff --git a/test/hotspot/jtreg/vmTestbase/nsk/stress/strace/strace009.java b/test/hotspot/jtreg/vmTestbase/nsk/stress/strace/strace009.java
index b8dcbe93919..c5ac24fe2ae 100644
--- a/test/hotspot/jtreg/vmTestbase/nsk/stress/strace/strace009.java
+++ b/test/hotspot/jtreg/vmTestbase/nsk/stress/strace/strace009.java
@@ -75,6 +75,7 @@ public class strace009 {
     static final int SLEEP_TIME = 50;
     static final String[] EXPECTED_METHODS = {
             "java.lang.Thread.sleep",
+            "java.lang.Thread.sleep0",
             "nsk.stress.strace.strace009Thread.run",
             "nsk.stress.strace.strace009Thread.recursiveMethod1",
             "nsk.stress.strace.strace009Thread.recursiveMethod2"
diff --git a/test/hotspot/jtreg/vmTestbase/nsk/stress/strace/strace010.java b/test/hotspot/jtreg/vmTestbase/nsk/stress/strace/strace010.java
index ebf07bd310a..e373212f40a 100644
--- a/test/hotspot/jtreg/vmTestbase/nsk/stress/strace/strace010.java
+++ b/test/hotspot/jtreg/vmTestbase/nsk/stress/strace/strace010.java
@@ -71,6 +71,7 @@ public class strace010 {
     static final int THRD_COUNT = 100;
     static final String[] EXPECTED_METHODS = {
             "java.lang.Thread.sleep",
+            "java.lang.Thread.sleep0",
             "nsk.stress.strace.strace010Thread.run",
             "nsk.stress.strace.strace010Thread.recursiveMethod"
     };
diff --git a/test/hotspot/jtreg/vmTestbase/nsk/stress/strace/strace011.java b/test/hotspot/jtreg/vmTestbase/nsk/stress/strace/strace011.java
index f18ca08e387..82a25f5937d 100644
--- a/test/hotspot/jtreg/vmTestbase/nsk/stress/strace/strace011.java
+++ b/test/hotspot/jtreg/vmTestbase/nsk/stress/strace/strace011.java
@@ -72,6 +72,7 @@ public class strace011 {
     static final int THRD_COUNT = 50;
     static final String[] EXPECTED_METHODS = {
             "java.lang.Thread.sleep",
+            "java.lang.Thread.sleep0",
             "nsk.stress.strace.strace011Thread.run",
             "nsk.stress.strace.strace011Thread.recursiveMethod"
     };
diff --git a/test/hotspot/jtreg/vmTestbase/nsk/stress/strace/strace012.java b/test/hotspot/jtreg/vmTestbase/nsk/stress/strace/strace012.java
index ad14f71cf88..cf11de1ce4e 100644
--- a/test/hotspot/jtreg/vmTestbase/nsk/stress/strace/strace012.java
+++ b/test/hotspot/jtreg/vmTestbase/nsk/stress/strace/strace012.java
@@ -75,6 +75,7 @@ public class strace012 {
     static final int THRD_COUNT = 100;
     static final String[] EXPECTED_METHODS = {
             "java.lang.Thread.sleep",
+            "java.lang.Thread.sleep0",
             "nsk.stress.strace.strace012Thread.run",
             "nsk.stress.strace.strace012Thread.recursiveMethod1",
             "nsk.stress.strace.strace012Thread.recursiveMethod2"
diff --git a/test/jdk/com/alibaba/wisp/ExecutionTest.java b/test/jdk/com/alibaba/wisp/ExecutionTest.java
index 35742fdf676..2738f418015 100644
--- a/test/jdk/com/alibaba/wisp/ExecutionTest.java
+++ b/test/jdk/com/alibaba/wisp/ExecutionTest.java
@@ -1,7 +1,7 @@
 /*
  * @test
  * @summary Test WispEngine's multi-task schedule
- * @modules java.base/jdk.internal.misc
+ * @modules java.base/jdk.internal.access
  * @run main/othervm -XX:+EnableCoroutine -Dcom.alibaba.wisp.transparentWispSwitch=true ExecutionTest
  * @run main/othervm -XX:+EnableCoroutine -Dcom.alibaba.wisp.transparentWispSwitch=true -Dcom.alibaba.wisp.version=2 ExecutionTest
 */
diff --git a/test/jdk/com/alibaba/wisp/IoTest.java b/test/jdk/com/alibaba/wisp/IoTest.java
index 5fa35f0dff6..f89881a6174 100644
--- a/test/jdk/com/alibaba/wisp/IoTest.java
+++ b/test/jdk/com/alibaba/wisp/IoTest.java
@@ -1,7 +1,7 @@
 /*
  * @test
  * @summary Test Wisp engine's NIO support
- * @modules java.base/jdk.internal.misc
+ * @modules java.base/jdk.internal.access
  * @run main/othervm -XX:+EnableCoroutine IoTest
  * @run main/othervm -XX:+EnableCoroutine -Dcom.alibaba.wisp.version=2 IoTest
 */
diff --git a/test/jdk/com/alibaba/wisp/ParkTest.java b/test/jdk/com/alibaba/wisp/ParkTest.java
index ef2e07b2a78..8cdc6dc1396 100644
--- a/test/jdk/com/alibaba/wisp/ParkTest.java
+++ b/test/jdk/com/alibaba/wisp/ParkTest.java
@@ -1,7 +1,7 @@
 /*
  * @test
  * @summary Test Wisp engine park / unpark
- * @modules java.base/jdk.internal.misc
+ * @modules java.base/jdk.internal.access
  * @run main/othervm -XX:+EnableCoroutine ParkTest
  * @run main/othervm -XX:+EnableCoroutine -Dcom.alibaba.wisp.version=2 ParkTest
 */
diff --git a/test/jdk/com/alibaba/wisp/SelectorLazyCreateTest.java b/test/jdk/com/alibaba/wisp/SelectorLazyCreateTest.java
index 033ac0a9066..5abd4838fce 100644
--- a/test/jdk/com/alibaba/wisp/SelectorLazyCreateTest.java
+++ b/test/jdk/com/alibaba/wisp/SelectorLazyCreateTest.java
@@ -1,6 +1,6 @@
 /*
  * @test
- * @library /lib/testlibrary
+ * @library /test/lib
  * @summary Test for engine.selector lazy created
  * @modules java.base/com.alibaba.wisp.engine:+open
  * @run main/othervm -XX:+EnableCoroutine -Dcom.alibaba.wisp.transparentWispSwitch=true -Dcom.alibaba.globalPoller=false SelectorLazyCreateTest
@@ -11,8 +11,8 @@
 import java.lang.reflect.Field;
 import java.net.Socket;
 
-import static jdk.testlibrary.Asserts.assertNotNull;
-import static jdk.testlibrary.Asserts.assertNull;
+import static jdk.test.lib.Asserts.assertNotNull;
+import static jdk.test.lib.Asserts.assertNull;
 
 public class SelectorLazyCreateTest {
 
diff --git a/test/jdk/com/alibaba/wisp/boot/UnsafeDependencyBugTest.java b/test/jdk/com/alibaba/wisp/boot/UnsafeDependencyBugTest.java
index 5fa9036da0e..2121f28b5aa 100644
--- a/test/jdk/com/alibaba/wisp/boot/UnsafeDependencyBugTest.java
+++ b/test/jdk/com/alibaba/wisp/boot/UnsafeDependencyBugTest.java
@@ -1,6 +1,6 @@
 /*
  * @test
- * @library /lib/testlibrary
+ * @library /test/lib
  * @summary test bug fix of SharedSecrets and Unsafe class initializer circular dependency
  * @run main UnsafeDependencyBugTest 10
  */
@@ -9,7 +9,7 @@
 import java.io.File;
 import java.util.concurrent.TimeUnit;
 
-import static jdk.testlibrary.Asserts.assertTrue;
+import static jdk.test.lib.Asserts.assertTrue;
 
 /**
  * We need thousand times to reproduce the DEADLOCK. Don't spend too much time here..
diff --git a/test/jdk/com/alibaba/wisp/bug/CancelTimerAndSleepTest.java b/test/jdk/com/alibaba/wisp/bug/CancelTimerAndSleepTest.java
index f6a79a70807..23831fb8b26 100644
--- a/test/jdk/com/alibaba/wisp/bug/CancelTimerAndSleepTest.java
+++ b/test/jdk/com/alibaba/wisp/bug/CancelTimerAndSleepTest.java
@@ -1,7 +1,7 @@
 /*
  * @test
  * @summary Test sleep
- * @modules java.base/jdk.internal.misc
+ * @modules java.base/jdk.internal.access
  * @run main/othervm -XX:+EnableCoroutine -Dcom.alibaba.wisp.transparentWispSwitch=true CancelTimerAndSleepTest
  * @run main/othervm -XX:+EnableCoroutine -Dcom.alibaba.wisp.transparentWispSwitch=true -Dcom.alibaba.wisp.version=2 CancelTimerAndSleepTest
 */
diff --git a/test/jdk/com/alibaba/wisp/bug/ClearEventTest.java b/test/jdk/com/alibaba/wisp/bug/ClearEventTest.java
index da37806ddac..e771fbc6ccd 100644
--- a/test/jdk/com/alibaba/wisp/bug/ClearEventTest.java
+++ b/test/jdk/com/alibaba/wisp/bug/ClearEventTest.java
@@ -1,7 +1,7 @@
 /*
  * @test
  * @summary Explain the fix of T11748781
- * @modules java.base/jdk.internal.misc
+ * @modules java.base/jdk.internal.access
  * @modules java.base/com.alibaba.wisp.engine:+open
  * @run main/othervm -XX:+EnableCoroutine -Dcom.alibaba.wisp.transparentWispSwitch=true ClearEventTest
  * @run main/othervm -XX:+EnableCoroutine -Dcom.alibaba.wisp.transparentWispSwitch=true -Dcom.alibaba.wisp.version=2 ClearEventTest
diff --git a/test/jdk/com/alibaba/wisp/bug/Id2TaskMapLeakTest.java b/test/jdk/com/alibaba/wisp/bug/Id2TaskMapLeakTest.java
index b93a1e8292d..6e6638e209e 100644
--- a/test/jdk/com/alibaba/wisp/bug/Id2TaskMapLeakTest.java
+++ b/test/jdk/com/alibaba/wisp/bug/Id2TaskMapLeakTest.java
@@ -1,6 +1,6 @@
 /*
  * @test
- * @library /lib/testlibrary
+ * @library /test/lib
  * @summary Test for thread WispTask leak
  * @modules java.base/com.alibaba.wisp.engine:+open
  * @run main/othervm -XX:+EnableCoroutine -Dcom.alibaba.wisp.transparentWispSwitch=true Id2TaskMapLeakTest
@@ -13,7 +13,7 @@
 import java.util.Map;
 import java.util.concurrent.atomic.AtomicInteger;
 
-import static jdk.testlibrary.Asserts.assertEQ;
+import static jdk.test.lib.Asserts.assertEQ;
 
 public class Id2TaskMapLeakTest {
     public static void main(String[] args) throws Exception {
diff --git a/test/jdk/com/alibaba/wisp/bug/ReleaseWispSocketAndExitTest.java b/test/jdk/com/alibaba/wisp/bug/ReleaseWispSocketAndExitTest.java
index 9115b2a3385..33c9f454029 100644
--- a/test/jdk/com/alibaba/wisp/bug/ReleaseWispSocketAndExitTest.java
+++ b/test/jdk/com/alibaba/wisp/bug/ReleaseWispSocketAndExitTest.java
@@ -4,7 +4,7 @@
  *      1. task A fetch a socket S and release it.
  *      2. task B get the socket S and block on IO.
  *      3. task A exit and clean S's event, now B waiting forever...
- * @modules java.base/jdk.internal.misc
+ * @modules java.base/jdk.internal.access
  * @run main/othervm -XX:+EnableCoroutine -Dcom.alibaba.wisp.transparentWispSwitch=true -Dcom.alibaba.wisp.version=1 ReleaseWispSocketAndExitTest
  * @run main/othervm -XX:+EnableCoroutine -Dcom.alibaba.wisp.transparentWispSwitch=true -Dcom.alibaba.wisp.version=2 ReleaseWispSocketAndExitTest
 */
diff --git a/test/jdk/com/alibaba/wisp/bug/ResetTaskCancelTimerBugTest.java b/test/jdk/com/alibaba/wisp/bug/ResetTaskCancelTimerBugTest.java
index 34a44805294..40e6e72e2c6 100644
--- a/test/jdk/com/alibaba/wisp/bug/ResetTaskCancelTimerBugTest.java
+++ b/test/jdk/com/alibaba/wisp/bug/ResetTaskCancelTimerBugTest.java
@@ -1,6 +1,6 @@
 /*
  * @test
- * @library /lib/testlibrary
+ * @library /test/lib
  * @summary test reset task doesn't cancel the current task's timer unexpectedly.
  * @run main/othervm -XX:+EnableCoroutine -Dcom.alibaba.wisp.transparentWispSwitch=true -Dcom.alibaba.wisp.version=1 ResetTaskCancelTimerBugTest
  */
@@ -12,7 +12,7 @@
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicReference;
 
-import static jdk.testlibrary.Asserts.assertTrue;
+import static jdk.test.lib.Asserts.assertTrue;
 
 
 public class ResetTaskCancelTimerBugTest {
diff --git a/test/jdk/com/alibaba/wisp/bug/SelectorInitCriticalTest.java b/test/jdk/com/alibaba/wisp/bug/SelectorInitCriticalTest.java
deleted file mode 100644
index fa7c82c2cb7..00000000000
--- a/test/jdk/com/alibaba/wisp/bug/SelectorInitCriticalTest.java
+++ /dev/null
@@ -1,32 +0,0 @@
-/*
- * @test
- * @library /lib/testlibrary
- * @summary Test the fix to NPE issue caused by unexpected co-routine yielding on synchronized(lock) in SelectorProvider.provider() during initialization of WispEngine
- * @run main/othervm -XX:+EnableCoroutine -Dcom.alibaba.wisp.transparentWispSwitch=true -XX:+UseWispMonitor SelectorInitCriticalTest
- * @run main/othervm -XX:+EnableCoroutine -Dcom.alibaba.wisp.transparentWispSwitch=true -XX:+UseWispMonitor -Dcom.alibaba.wisp.version=2 SelectorInitCriticalTest
-*/
-
-
-import java.lang.reflect.Field;
-import java.nio.channels.spi.SelectorProvider;
-import java.util.concurrent.CountDownLatch;
-
-public class SelectorInitCriticalTest {
-    public static void main(String[] args) throws Exception {
-        Field f = SelectorProvider.class.getDeclaredField("lock");
-        f.setAccessible(true);
-        Object selectorProviderLock = f.get(null);
-        CountDownLatch latch = new CountDownLatch(1);
-
-        Thread t = new Thread(latch::countDown);
-
-        synchronized (selectorProviderLock) {
-            t.start();
-            // Holding selectorProviderLock for a while which will eventually blocks the initialization of t' WispEngine
-            Thread.sleep(100);
-        }
-
-        latch.await();
-
-    }
-}
diff --git a/test/jdk/com/alibaba/wisp/bug/TestThreadStackTrace.sh b/test/jdk/com/alibaba/wisp/bug/TestThreadStackTrace.sh
index 740cc98444d..810504f94bc 100644
--- a/test/jdk/com/alibaba/wisp/bug/TestThreadStackTrace.sh
+++ b/test/jdk/com/alibaba/wisp/bug/TestThreadStackTrace.sh
@@ -2,7 +2,7 @@
 #
 # @test
 # @summary test Thread.getStackTrace() in wisp transparentAsync model
-# @modules java.base/jdk.internal.misc
+# @modules java.base/jdk.internal.access
 # @modules java.base/com.alibaba.wisp.engine:+open
 # @run shell TestThreadStackTrace.sh
 #
@@ -107,7 +107,7 @@ public class TmpThreadStackTrace {
 EOF
 
 # Do compilation
-${JAVAC} --add-exports java.base/jdk.internal.misc=ALL-UNNAMED -cp ${TESTCLASSES} -d ${TESTCLASSES} ${TESTCLASSES}${FS}$TEST_SOURCE >> /dev/null 2>&1
+${JAVAC} --add-exports java.base/jdk.internal.access=ALL-UNNAMED -cp ${TESTCLASSES} -d ${TESTCLASSES} ${TESTCLASSES}${FS}$TEST_SOURCE >> /dev/null 2>&1
 if [ $? != '0' ]
 then
 	printf "Failed to compile ${TESTCLASSES}${FS}${TEST_SOURCE}"
@@ -115,7 +115,7 @@ then
 fi
 
 #run
-${JAVA} -XX:+PrintSafepointStatistics  -XX:PrintSafepointStatisticsCount=1 -Dcom.alibaba.wisp.config=${TEST_WISP_CONFIG} -XX:-UseBiasedLocking -XX:+EnableCoroutine -Dcom.alibaba.wisp.transparentWispSwitch=true  -Dcom.alibaba.wisp.enableThreadAsWisp=true -cp ${TESTCLASSES} ${TEST_CLASS} > output.txt  2>&1
+${JAVA} -Xlog:safepoint+stats=debug -Dcom.alibaba.wisp.config=${TEST_WISP_CONFIG} -XX:-UseBiasedLocking -XX:+EnableCoroutine -Dcom.alibaba.wisp.transparentWispSwitch=true  -Dcom.alibaba.wisp.enableThreadAsWisp=true -cp ${TESTCLASSES} ${TEST_CLASS} > output.txt  2>&1
 rm -f $TEST_WISP_CONFIG
 cat output.txt
 
diff --git a/test/jdk/com/alibaba/wisp/bug/ThreadLockTest.java b/test/jdk/com/alibaba/wisp/bug/ThreadLockTest.java
index 81fe5def1e2..ca7e56a0f23 100644
--- a/test/jdk/com/alibaba/wisp/bug/ThreadLockTest.java
+++ b/test/jdk/com/alibaba/wisp/bug/ThreadLockTest.java
@@ -2,8 +2,8 @@
  * @test
  * @summary Test fix of WispEngine block on Thread.class lock
  * @modules java.base/java.lang:+open
- * @run main/othervm -XX:-UseBiasedLocking -XX:+EnableCoroutine -XX:+UseWispMonitor -XX:+UnlockExperimentalVMOptions -XX:SyncKnobs="ReportSettings=1:QMode=1" -Dcom.alibaba.wisp.transparentWispSwitch=true ThreadLockTest
- * @run main/othervm -XX:-UseBiasedLocking -XX:+EnableCoroutine -XX:+UseWispMonitor -XX:+UnlockExperimentalVMOptions -XX:SyncKnobs="ReportSettings=1:QMode=1" -Dcom.alibaba.wisp.transparentWispSwitch=true -Dcom.alibaba.wisp.version=2 ThreadLockTest
+ * @run main/othervm -XX:-UseBiasedLocking -XX:+EnableCoroutine -XX:+UseWispMonitor -Dcom.alibaba.wisp.transparentWispSwitch=true ThreadLockTest
+ * @run main/othervm -XX:-UseBiasedLocking -XX:+EnableCoroutine -XX:+UseWispMonitor -Dcom.alibaba.wisp.transparentWispSwitch=true -Dcom.alibaba.wisp.version=2 ThreadLockTest
  */
 
 import com.alibaba.wisp.engine.WispEngine;
diff --git a/test/jdk/com/alibaba/wisp/bug/WispEngineCriticalSectionTest.java b/test/jdk/com/alibaba/wisp/bug/WispEngineCriticalSectionTest.java
index f96f46ebd04..68c62fb2eb1 100644
--- a/test/jdk/com/alibaba/wisp/bug/WispEngineCriticalSectionTest.java
+++ b/test/jdk/com/alibaba/wisp/bug/WispEngineCriticalSectionTest.java
@@ -1,8 +1,9 @@
 /*
  * @test
  * @summary Test case for fix a deadlock caused by critical section of WispEngine
- * @modules java.base/jdk.internal.misc
+ * @modules java.base/jdk.internal.access
  * @modules java.base/com.alibaba.wisp.engine:+open
+ * @modules java.base/java.nio.channels.spi:+open
  * @run main/othervm   -XX:-UseBiasedLocking -XX:+EnableCoroutine -XX:+UseWispMonitor -Dcom.alibaba.wisp.transparentWispSwitch=true -Dcom.alibaba.globalPoller=false WispEngineCriticalSectionTest
  */
 
@@ -14,6 +15,7 @@
 import java.net.Socket;
 import java.nio.channels.spi.AbstractSelector;
 import java.util.HashSet;
+import java.util.Set;
 import java.util.concurrent.CountDownLatch;
 
 public class WispEngineCriticalSectionTest {
@@ -31,8 +33,8 @@ public static void main(String[] args) throws Exception {
         AbstractSelector selector = (AbstractSelector) f.get(WispEngine.current());
         f = AbstractSelector.class.getDeclaredField("cancelledKeys");
         f.setAccessible(true);
-        final HashSet cancelSet = (HashSet) f.get(selector);
-
+        Set set = (Set)(f.get(selector));
+        final HashSet cancelSet = new HashSet(set);
 
         // made an runnable wisp
         CountDownLatch cd = new CountDownLatch(1);
diff --git a/test/jdk/com/alibaba/wisp/bug/WispSocketLeakWhenConnectTimeoutTest.java b/test/jdk/com/alibaba/wisp/bug/WispSocketLeakWhenConnectTimeoutTest.java
index 725832626f8..7af3919a6c0 100644
--- a/test/jdk/com/alibaba/wisp/bug/WispSocketLeakWhenConnectTimeoutTest.java
+++ b/test/jdk/com/alibaba/wisp/bug/WispSocketLeakWhenConnectTimeoutTest.java
@@ -1,6 +1,6 @@
 /*
  * @test
- * @library /lib/testlibrary
+ * @library /test/lib
  * @summary test the fix to fd leakage when socket connect timeout
  * @run main/othervm -XX:+EnableCoroutine -Dcom.alibaba.wisp.transparentWispSwitch=true -Dcom.alibaba.wisp.version=2 WispSocketLeakWhenConnectTimeoutTest
 */
@@ -10,7 +10,7 @@
 import java.net.Socket;
 import java.net.SocketTimeoutException;
 
-import static jdk.testlibrary.Asserts.assertTrue;
+import static jdk.test.lib.Asserts.assertTrue;
 
 public class WispSocketLeakWhenConnectTimeoutTest {
     public static void main(String[] args) throws IOException {
diff --git a/test/jdk/com/alibaba/wisp/env/CtxClassLoaderIsolateTest.java b/test/jdk/com/alibaba/wisp/env/CtxClassLoaderIsolateTest.java
index 9adb6d19e98..a4df7176f2c 100644
--- a/test/jdk/com/alibaba/wisp/env/CtxClassLoaderIsolateTest.java
+++ b/test/jdk/com/alibaba/wisp/env/CtxClassLoaderIsolateTest.java
@@ -1,6 +1,6 @@
 /*
  * @test
- * @library /lib/testlibrary
+ * @library /test/lib
  * @summary Verify the context class loader isolation per co-routine
  * @run main/othervm -XX:+EnableCoroutine -Dcom.alibaba.wisp.transparentWispSwitch=true CtxClassLoaderIsolateTest
  * @run main/othervm -XX:+EnableCoroutine -Dcom.alibaba.wisp.transparentWispSwitch=true -Dcom.alibaba.wisp.version=2 CtxClassLoaderIsolateTest
@@ -9,7 +9,7 @@
 
 import com.alibaba.wisp.engine.WispEngine;
 
-import static jdk.testlibrary.Asserts.assertTrue;
+import static jdk.test.lib.Asserts.assertTrue;
 
 public class CtxClassLoaderIsolateTest {
     public static void main(String[] args) throws Exception {
diff --git a/test/jdk/com/alibaba/wisp/io/CreateFdOnDemandTest.java b/test/jdk/com/alibaba/wisp/io/CreateFdOnDemandTest.java
index e9731ac78f7..389cdd45797 100644
--- a/test/jdk/com/alibaba/wisp/io/CreateFdOnDemandTest.java
+++ b/test/jdk/com/alibaba/wisp/io/CreateFdOnDemandTest.java
@@ -1,6 +1,6 @@
 /*
  * @test
- * @library /lib/testlibrary
+ * @library /test/lib
  * @summary Test fix of unconnected Socket fd leak.
  * @run main/othervm -XX:+EnableCoroutine -Dcom.alibaba.wisp.transparentWispSwitch=true CreateFdOnDemandTest
  * @run main/othervm -XX:+EnableCoroutine -Dcom.alibaba.wisp.transparentWispSwitch=true -Dcom.alibaba.wisp.version=2 CreateFdOnDemandTest
@@ -13,8 +13,8 @@
 
 import com.alibaba.wisp.engine.WispEngine;
 
-import static jdk.testlibrary.Asserts.assertEQ;
-import static jdk.testlibrary.Asserts.assertTrue;
+import static jdk.test.lib.Asserts.assertEQ;
+import static jdk.test.lib.Asserts.assertTrue;
 
 public class CreateFdOnDemandTest {
     public static void main(String[] args) throws Exception {
diff --git a/test/jdk/com/alibaba/wisp/io/DatagramSocketTest.java b/test/jdk/com/alibaba/wisp/io/DatagramSocketTest.java
index e580720def0..886d389de93 100644
--- a/test/jdk/com/alibaba/wisp/io/DatagramSocketTest.java
+++ b/test/jdk/com/alibaba/wisp/io/DatagramSocketTest.java
@@ -1,8 +1,8 @@
 /*
  * @test
- * @library /lib/testlibrary
+ * @library /test/lib
  * @summary Test WispEngine's DatagramSocket, InitialDirContext use dup socket to query dns.
- * @modules java.base/jdk.internal.misc
+ * @modules java.base/jdk.internal.access
  * @run main/othervm -XX:+EnableCoroutine -Dcom.alibaba.wisp.transparentWispSwitch=true DatagramSocketTest
  * @run main/othervm -XX:+EnableCoroutine -Dcom.alibaba.wisp.transparentWispSwitch=true -Dcom.alibaba.wisp.version=2 DatagramSocketTest
 */
@@ -15,7 +15,7 @@
 import java.io.IOException;
 import java.net.*;
 import java.util.concurrent.CountDownLatch;
-import static jdk.testlibrary.Asserts.*;
+import static jdk.test.lib.Asserts.*;
 
 public class DatagramSocketTest {
     public static void main(String[] args) throws Exception {
diff --git a/test/jdk/com/alibaba/wisp/io/GlobalPollerTest.java b/test/jdk/com/alibaba/wisp/io/GlobalPollerTest.java
index 9189516f4f2..26bf2aba76d 100644
--- a/test/jdk/com/alibaba/wisp/io/GlobalPollerTest.java
+++ b/test/jdk/com/alibaba/wisp/io/GlobalPollerTest.java
@@ -1,8 +1,8 @@
 /*
  * @test
- * @library /lib/testlibrary
+ * @library /test/lib
  * @summary Test for Global Poller
- * @modules java.base/jdk.internal.misc
+ * @modules java.base/jdk.internal.access
  * @modules java.base/sun.nio.ch
  * @modules java.base/com.alibaba.wisp.engine:+open
  * @run main/othervm -XX:+EnableCoroutine -Dcom.alibaba.transparentAsync=true GlobalPollerTest
@@ -20,7 +20,7 @@
 import java.nio.channels.SelectionKey;
 import java.nio.channels.SocketChannel;
 
-import static jdk.testlibrary.Asserts.assertTrue;
+import static jdk.test.lib.Asserts.assertTrue;
 
 public class GlobalPollerTest {
     private static WispEngineAccess access = SharedSecrets.getWispEngineAccess();
diff --git a/test/jdk/com/alibaba/wisp/io/ReuseUdpSocektBufTest.java b/test/jdk/com/alibaba/wisp/io/ReuseUdpSocektBufTest.java
index 44ceceecca4..0f46625e1d8 100644
--- a/test/jdk/com/alibaba/wisp/io/ReuseUdpSocektBufTest.java
+++ b/test/jdk/com/alibaba/wisp/io/ReuseUdpSocektBufTest.java
@@ -1,6 +1,6 @@
 /*
  * @test
- * @library /lib/testlibrary
+ * @library /test/lib
  * @summary test reuse WispUdpSocket buffer
  * @run main/othervm -XX:+EnableCoroutine -Dcom.alibaba.wisp.transparentWispSwitch=true ReuseUdpSocektBufTest
  */
@@ -9,7 +9,7 @@
 import java.net.DatagramSocket;
 import java.net.InetAddress;
 
-import static jdk.testlibrary.Asserts.assertTrue;
+import static jdk.test.lib.Asserts.assertTrue;
 
 
 public class ReuseUdpSocektBufTest {
diff --git a/test/jdk/com/alibaba/wisp/lock/AQSTest.java b/test/jdk/com/alibaba/wisp/lock/AQSTest.java
index cdba42525e8..5d4d6a6bf4c 100644
--- a/test/jdk/com/alibaba/wisp/lock/AQSTest.java
+++ b/test/jdk/com/alibaba/wisp/lock/AQSTest.java
@@ -1,7 +1,7 @@
 /*
  * @test
  * @summary Test AQS: CountDownLatch is implement by AQS
- * @modules java.base/jdk.internal.misc
+ * @modules java.base/jdk.internal.access
  * @run main/othervm -XX:+EnableCoroutine -Dcom.alibaba.wisp.transparentWispSwitch=true AQSTest
  * @run main/othervm -XX:+EnableCoroutine -Dcom.alibaba.wisp.transparentWispSwitch=true -Dcom.alibaba.wisp.version=2 AQSTest
 */
diff --git a/test/jdk/com/alibaba/wisp/lock/ElisionSpinTest.java b/test/jdk/com/alibaba/wisp/lock/ElisionSpinTest.java
index 4428d7a198a..e56e02072da 100644
--- a/test/jdk/com/alibaba/wisp/lock/ElisionSpinTest.java
+++ b/test/jdk/com/alibaba/wisp/lock/ElisionSpinTest.java
@@ -1,8 +1,8 @@
 /*
  * @test
  * @summary Test elision spin
- * @modules java.base/jdk.internal.misc
- * @library /lib/testlibrary
+ * @modules java.base/jdk.internal.access
+ * @library /test/lib
  * @run main/othervm  -XX:+EnableCoroutine -Dcom.alibaba.wisp.transparentWispSwitch=true -Dcom.alibaba.wisp.useStealLock=false ElisionSpinTest
  */
 
@@ -12,8 +12,8 @@
 import java.util.concurrent.locks.Condition;
 import java.util.concurrent.locks.ReentrantLock;
 
-import static jdk.testlibrary.Asserts.assertFalse;
-import static jdk.testlibrary.Asserts.assertTrue;
+import static jdk.test.lib.Asserts.assertFalse;
+import static jdk.test.lib.Asserts.assertTrue;
 
 public class ElisionSpinTest {
     public static void main(String[] args) {
diff --git a/test/jdk/com/alibaba/wisp/lock/LockTest.java b/test/jdk/com/alibaba/wisp/lock/LockTest.java
index 18dd446cf3c..bdc74598028 100644
--- a/test/jdk/com/alibaba/wisp/lock/LockTest.java
+++ b/test/jdk/com/alibaba/wisp/lock/LockTest.java
@@ -1,7 +1,7 @@
 /*
  * @test
  * @summary Test ReentrantLock in coroutine environment
- * @modules java.base/jdk.internal.misc
+ * @modules java.base/jdk.internal.access
  * @run main/othervm  -XX:+EnableCoroutine  -Dcom.alibaba.wisp.transparentWispSwitch=true LockTest
  * @run main/othervm  -XX:+EnableCoroutine  -Dcom.alibaba.wisp.transparentWispSwitch=true -Dcom.alibaba.wisp.version=2 LockTest
 */
diff --git a/test/jdk/com/alibaba/wisp/lock/LockUninterruptiblyTest.java b/test/jdk/com/alibaba/wisp/lock/LockUninterruptiblyTest.java
index 4cc9e34ca70..58cffa63ac4 100644
--- a/test/jdk/com/alibaba/wisp/lock/LockUninterruptiblyTest.java
+++ b/test/jdk/com/alibaba/wisp/lock/LockUninterruptiblyTest.java
@@ -1,6 +1,6 @@
 /*
  * @test
- * @library /lib/testlibrary
+ * @library /test/lib
  * @summary Test to verify we are not spinning when we're trying to acquire monitor with interrupted status
  * @modules java.base/com.alibaba.wisp.engine:+open
  * @run main/othervm -XX:+EnableCoroutine -XX:+UseWispMonitor -Dcom.alibaba.wisp.transparentWispSwitch=true LockUninterruptiblyTest
@@ -12,7 +12,7 @@
 import java.lang.reflect.Field;
 import java.util.concurrent.atomic.AtomicReference;
 
-import static jdk.testlibrary.Asserts.assertTrue;
+import static jdk.test.lib.Asserts.assertTrue;
 
 public class LockUninterruptiblyTest {
 
diff --git a/test/jdk/com/alibaba/wisp/lock/UnsafeParkTest.java b/test/jdk/com/alibaba/wisp/lock/UnsafeParkTest.java
index 2edd745b01f..3945df55e5a 100644
--- a/test/jdk/com/alibaba/wisp/lock/UnsafeParkTest.java
+++ b/test/jdk/com/alibaba/wisp/lock/UnsafeParkTest.java
@@ -1,6 +1,6 @@
 /*
  * @test
- * @library /lib/testlibrary
+ * @library /test/lib
  * @summary Test to verify we can do proper wisp scheduling while calling on Unsafe.park()
  * @run main/othervm -XX:+EnableCoroutine -Dcom.alibaba.wisp.transparentWispSwitch=true UnsafeParkTest
  * @run main/othervm -XX:+EnableCoroutine -Dcom.alibaba.wisp.transparentWispSwitch=true -Dcom.alibaba.wisp.version=2 UnsafeParkTest
@@ -13,7 +13,7 @@
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicLong;
 
-import static jdk.testlibrary.Asserts.assertTrue;
+import static jdk.test.lib.Asserts.assertTrue;
 
 
 public class UnsafeParkTest {
diff --git a/test/jdk/com/alibaba/wisp/monitor/JNICriticalTest.java b/test/jdk/com/alibaba/wisp/monitor/JNICriticalTest.java
index 394bd9909e5..da033e43829 100644
--- a/test/jdk/com/alibaba/wisp/monitor/JNICriticalTest.java
+++ b/test/jdk/com/alibaba/wisp/monitor/JNICriticalTest.java
@@ -1,7 +1,7 @@
 /*
  * @test
  * @summary Test unpark in JNI critical case
- * @modules java.base/jdk.internal.misc
+ * @modules java.base/jdk.internal.access
  * @run main/othervm   -XX:-UseBiasedLocking -XX:+EnableCoroutine -XX:+UseWispMonitor -Dcom.alibaba.wisp.transparentWispSwitch=true -Dcom.alibaba.wisp.version=2 JNICriticalTest
  */
 
diff --git a/test/jdk/com/alibaba/wisp/monitor/LazyUnparkBugTest.java b/test/jdk/com/alibaba/wisp/monitor/LazyUnparkBugTest.java
index 59489f242d2..008408cf8df 100644
--- a/test/jdk/com/alibaba/wisp/monitor/LazyUnparkBugTest.java
+++ b/test/jdk/com/alibaba/wisp/monitor/LazyUnparkBugTest.java
@@ -1,7 +1,7 @@
 /*
  * @test
  * @summary T12212948
- * @library /lib/testlibrary
+ * @library /test/lib
  * @run main/othervm -XX:+EnableCoroutine -XX:+UseWispMonitor -Dcom.alibaba.wisp.transparentWispSwitch=true LazyUnparkBugTest
  * @run main/othervm -XX:+EnableCoroutine -XX:+UseWispMonitor -Dcom.alibaba.wisp.transparentWispSwitch=true -Dcom.alibaba.wisp.version=2 LazyUnparkBugTest
 */
diff --git a/test/jdk/com/alibaba/wisp/monitor/MultiThreadTest.java b/test/jdk/com/alibaba/wisp/monitor/MultiThreadTest.java
index 0621411bcd6..851268e1cb7 100644
--- a/test/jdk/com/alibaba/wisp/monitor/MultiThreadTest.java
+++ b/test/jdk/com/alibaba/wisp/monitor/MultiThreadTest.java
@@ -1,8 +1,8 @@
 /*
  * @test
- * @library /lib/testlibrary
+ * @library /test/lib
  * @summary Test object lock with coroutine
- * @modules java.base/jdk.internal.misc
+ * @modules java.base/jdk.internal.access
  * @run main/othervm  -XX:-UseBiasedLocking -XX:+EnableCoroutine -XX:+UseWispMonitor -Dcom.alibaba.wisp.transparentWispSwitch=true -Dcom.alibaba.wisp.version=2 MultiThreadTest
 */
 
diff --git a/test/jdk/com/alibaba/wisp/monitor/WaitNotifyTest.java b/test/jdk/com/alibaba/wisp/monitor/WaitNotifyTest.java
index dbbb7c3088d..668f76441ac 100644
--- a/test/jdk/com/alibaba/wisp/monitor/WaitNotifyTest.java
+++ b/test/jdk/com/alibaba/wisp/monitor/WaitNotifyTest.java
@@ -1,6 +1,6 @@
 /*
  * @test
- * @library /lib/testlibrary
+ * @library /test/lib
  * @summary Test Object.wait/notify with coroutine
  * @run main/othervm  -XX:-UseBiasedLocking -XX:+EnableCoroutine -XX:+UseWispMonitor -Dcom.alibaba.wisp.transparentWispSwitch=true WaitNotifyTest
  */
@@ -9,7 +9,7 @@
 
 import java.util.concurrent.CountDownLatch;
 
-import static jdk.testlibrary.Asserts.assertEQ;
+import static jdk.test.lib.Asserts.assertEQ;
 
 public class WaitNotifyTest {
     public static void main(String[] args) throws Exception {
diff --git a/test/jdk/com/alibaba/wisp/thread/DisableThreadAsWispAtRuntimeTest.java b/test/jdk/com/alibaba/wisp/thread/DisableThreadAsWispAtRuntimeTest.java
index b1eeb4e0080..028216e23be 100644
--- a/test/jdk/com/alibaba/wisp/thread/DisableThreadAsWispAtRuntimeTest.java
+++ b/test/jdk/com/alibaba/wisp/thread/DisableThreadAsWispAtRuntimeTest.java
@@ -1,9 +1,9 @@
 /*
  * @test
  * @summary test to turn on/off shiftThreadModel on demand
- * @modules java.base/jdk.internal.misc
+ * @modules java.base/jdk.internal.access
  * @modules java.base/com.alibaba.wisp.engine:+open
- * @library /lib/testlibrary
+ * @library /test/lib
  * @run main/othervm -XX:+EnableCoroutine -XX:+UseWispMonitor -Dcom.alibaba.wisp.transparentWispSwitch=true DisableThreadAsWispAtRuntimeTest
  * @run main/othervm -XX:+EnableCoroutine -XX:+UseWispMonitor -Dcom.alibaba.wisp.transparentWispSwitch=true -Dcom.alibaba.wisp.version=2 DisableThreadAsWispAtRuntimeTest
 */
@@ -19,7 +19,7 @@
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
 
-import static jdk.testlibrary.Asserts.assertEQ;
+import static jdk.test.lib.Asserts.assertEQ;
 
 public class DisableThreadAsWispAtRuntimeTest {
 
diff --git a/test/jdk/com/alibaba/wisp/thread/EngineExecutorTest.java b/test/jdk/com/alibaba/wisp/thread/EngineExecutorTest.java
index 5fb13f2a312..f0473c8c42e 100644
--- a/test/jdk/com/alibaba/wisp/thread/EngineExecutorTest.java
+++ b/test/jdk/com/alibaba/wisp/thread/EngineExecutorTest.java
@@ -1,7 +1,7 @@
 /*
  * @test
  * @summary test submit task to engine.
- * @modules java.base/jdk.internal.misc
+ * @modules java.base/jdk.internal.access
  * @run main/othervm -XX:+EnableCoroutine -Dcom.alibaba.wisp.transparentWispSwitch=true EngineExecutorTest
  * @run main/othervm -XX:+EnableCoroutine -Dcom.alibaba.wisp.transparentWispSwitch=true -Dcom.alibaba.wisp.version=2 EngineExecutorTest 
 */
diff --git a/test/jdk/com/alibaba/wisp/thread/InterruptedSleepTest.java b/test/jdk/com/alibaba/wisp/thread/InterruptedSleepTest.java
index 68babde1f9f..6c705ab7bd4 100644
--- a/test/jdk/com/alibaba/wisp/thread/InterruptedSleepTest.java
+++ b/test/jdk/com/alibaba/wisp/thread/InterruptedSleepTest.java
@@ -1,16 +1,16 @@
 /*
  * @test
  * @summary test InterruptedException was thrown by sleep()
- * @library /lib/testlibrary
+ * @library /test/lib
  * @run main/othervm -XX:+EnableCoroutine -Dcom.alibaba.wisp.transparentWispSwitch=true InterruptedSleepTest
  * @run main/othervm -XX:+EnableCoroutine -Dcom.alibaba.wisp.transparentWispSwitch=true -Dcom.alibaba.wisp.version=2 InterruptedSleepTest
 */
 
 import com.alibaba.wisp.engine.WispEngine;
 
-import static jdk.testlibrary.Asserts.assertFalse;
-import static jdk.testlibrary.Asserts.assertLessThan;
-import static jdk.testlibrary.Asserts.assertTrue;
+import static jdk.test.lib.Asserts.assertFalse;
+import static jdk.test.lib.Asserts.assertLessThan;
+import static jdk.test.lib.Asserts.assertTrue;
 
 public class InterruptedSleepTest {
     public static void main(String[] args) {
diff --git a/test/jdk/com/alibaba/wisp/thread/IsAliveTest.java b/test/jdk/com/alibaba/wisp/thread/IsAliveTest.java
index 6b275cfbb92..b47ba747f0b 100644
--- a/test/jdk/com/alibaba/wisp/thread/IsAliveTest.java
+++ b/test/jdk/com/alibaba/wisp/thread/IsAliveTest.java
@@ -2,7 +2,7 @@
  * @test
  * @summary test thread.isAlive() of wispTask
  * @modules java.base/com.alibaba.wisp.engine:+open
- * @library /lib/testlibrary
+ * @library /test/lib
  * @run main/othervm -XX:+EnableCoroutine -Dcom.alibaba.wisp.transparentWispSwitch=true -Dcom.alibaba.wisp.enableThreadAsWisp=true IsAliveTest
  * @run main/othervm -XX:+EnableCoroutine -Dcom.alibaba.wisp.transparentWispSwitch=true -Dcom.alibaba.wisp.enableThreadAsWisp=true -Dcom.alibaba.wisp.version=2 IsAliveTest
  *
@@ -16,8 +16,8 @@
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicReference;
 
-import static jdk.testlibrary.Asserts.assertFalse;
-import static jdk.testlibrary.Asserts.assertTrue;
+import static jdk.test.lib.Asserts.assertFalse;
+import static jdk.test.lib.Asserts.assertTrue;
 
 
 public class IsAliveTest {
diff --git a/test/jdk/com/alibaba/wisp/thread/SubmittedTaskLimitTest.java b/test/jdk/com/alibaba/wisp/thread/SubmittedTaskLimitTest.java
index f1c09e31a76..cc625bbec37 100644
--- a/test/jdk/com/alibaba/wisp/thread/SubmittedTaskLimitTest.java
+++ b/test/jdk/com/alibaba/wisp/thread/SubmittedTaskLimitTest.java
@@ -2,7 +2,7 @@
  * @test
  * @summary Test to verify threshold setting for submitted wisp tasks.
  * @modules java.base/com.alibaba.wisp.engine:+open
- * @library /lib/testlibrary
+ * @library /test/lib
  * @run main/othervm -XX:+EnableCoroutine -Dcom.alibaba.wisp.transparentWispSwitch=true SubmittedTaskLimitTest
  */
 
@@ -14,9 +14,9 @@
 import java.util.concurrent.atomic.AtomicReference;
 import java.util.stream.Collectors;
 
-import static jdk.testlibrary.Asserts.assertGT;
-import static jdk.testlibrary.Asserts.assertGTE;
-import static jdk.testlibrary.Asserts.assertTrue;
+import static jdk.test.lib.Asserts.assertGT;
+import static jdk.test.lib.Asserts.assertGTE;
+import static jdk.test.lib.Asserts.assertTrue;
 
 
 public class SubmittedTaskLimitTest {
diff --git a/test/jdk/com/alibaba/wisp/thread/ThreadAsWispTest.java b/test/jdk/com/alibaba/wisp/thread/ThreadAsWispTest.java
index df2c222aa65..63b1627f314 100644
--- a/test/jdk/com/alibaba/wisp/thread/ThreadAsWispTest.java
+++ b/test/jdk/com/alibaba/wisp/thread/ThreadAsWispTest.java
@@ -1,9 +1,9 @@
 /*
  * @test
  * @summary test dispatching thread into our managed workers
- * @modules java.base/jdk.internal.misc
+ * @modules java.base/jdk.internal.access
  * @modules java.base/com.alibaba.wisp.engine:+open
- * @library /lib/testlibrary
+ * @library /test/lib
  * @run main/othervm -XX:+EnableCoroutine -XX:+UseWispMonitor -Dcom.alibaba.wisp.transparentWispSwitch=true -Dcom.alibaba.threadPoolLimit=true -Dcom.alibaba.wisp.enableThreadAsWisp=true ThreadAsWispTest
  * @run main/othervm -XX:+EnableCoroutine -XX:+UseWispMonitor -Dcom.alibaba.wisp.transparentWispSwitch=true -Dcom.alibaba.threadPoolLimit=true -Dcom.alibaba.wisp.enableThreadAsWisp=true -Dcom.alibaba.wisp.version=2 -XX:ActiveProcessorCount=4 ThreadAsWispTest
 */
@@ -30,8 +30,8 @@
 import java.util.function.BiConsumer;
 import java.util.stream.IntStream;
 
-import static jdk.testlibrary.Asserts.assertEQ;
-import static jdk.testlibrary.Asserts.assertTrue;
+import static jdk.test.lib.Asserts.assertEQ;
+import static jdk.test.lib.Asserts.assertTrue;
 
 public class ThreadAsWispTest {
     static Thread mainThread;
diff --git a/test/jdk/com/alibaba/wisp/thread/ThrowErrorTest.java b/test/jdk/com/alibaba/wisp/thread/ThrowErrorTest.java
index 373ceb31a8e..cc04e8a7f4d 100644
--- a/test/jdk/com/alibaba/wisp/thread/ThrowErrorTest.java
+++ b/test/jdk/com/alibaba/wisp/thread/ThrowErrorTest.java
@@ -1,14 +1,14 @@
 /*
  * @test
  * @summary test coroutine throw Error
- * @library /lib/testlibrary
+ * @library /test/lib
  * @run main/othervm -XX:+EnableCoroutine -Dcom.alibaba.wisp.transparentWispSwitch=true ThrowErrorTest
  * @run main/othervm -XX:+EnableCoroutine -Dcom.alibaba.wisp.transparentWispSwitch=true -Dcom.alibaba.wisp.version=2 ThrowErrorTest
 */
 
 import com.alibaba.wisp.engine.WispEngine;
 
-import static jdk.testlibrary.Asserts.assertTrue;
+import static jdk.test.lib.Asserts.assertTrue;
 
 public class ThrowErrorTest {
     public static void main(String[] args) {
diff --git a/test/jdk/com/alibaba/wisp/thread/TimeSliceTest.java b/test/jdk/com/alibaba/wisp/thread/TimeSliceTest.java
index d2c38f79e20..6f9fd078cc3 100644
--- a/test/jdk/com/alibaba/wisp/thread/TimeSliceTest.java
+++ b/test/jdk/com/alibaba/wisp/thread/TimeSliceTest.java
@@ -1,7 +1,7 @@
 /*
  * @test
  * @summary test wisp time slice
- * @library /lib/testlibrary
+ * @library /test/lib
  * @run main/othervm -XX:+EnableCoroutine  -XX:+EnableCoroutineTimeSlice -Dcom.alibaba.wisp.transparentWispSwitch=true -Dcom.alibaba.wisp.enableHandOff=true -Dcom.alibaba.wisp.sysmonTickUs=10000 -XX:-Inline -Dcom.alibaba.wisp.version=2 TimeSliceTest
 */
 
@@ -10,7 +10,7 @@
 import java.util.Date;
 import java.util.concurrent.atomic.AtomicReference;
 
-import static jdk.testlibrary.Asserts.assertTrue;
+import static jdk.test.lib.Asserts.assertTrue;
 
 public class TimeSliceTest {
     public static void main(String[] args) throws Exception {
diff --git a/test/jdk/com/alibaba/wisp/thread/YieldTest.java b/test/jdk/com/alibaba/wisp/thread/YieldTest.java
index 7a2ec67455e..5cd318991b1 100644
--- a/test/jdk/com/alibaba/wisp/thread/YieldTest.java
+++ b/test/jdk/com/alibaba/wisp/thread/YieldTest.java
@@ -1,13 +1,13 @@
 /*
  * @test
  * @summary test yield()
- * @library /lib/testlibrary
+ * @library /test/lib
  * @run main/othervm -XX:+EnableCoroutine -Dcom.alibaba.wisp.transparentWispSwitch=true YieldTest
  */
 
 import com.alibaba.wisp.engine.WispEngine;
 
-import static jdk.testlibrary.Asserts.*;
+import static jdk.test.lib.Asserts.*;
 
 public class YieldTest {
     private static int i = 0;
diff --git a/test/jdk/com/alibaba/wisp/timer/DaemonThreadGroupTest.java b/test/jdk/com/alibaba/wisp/timer/DaemonThreadGroupTest.java
index 82cf959998f..0c3248aa16e 100644
--- a/test/jdk/com/alibaba/wisp/timer/DaemonThreadGroupTest.java
+++ b/test/jdk/com/alibaba/wisp/timer/DaemonThreadGroupTest.java
@@ -1,8 +1,8 @@
 /*
  * @test
- * @library /lib/testlibrary
+ * @library /test/lib
  * @summary Test Daemon Thread Group implementation
- * @modules java.base/jdk.internal.misc
+ * @modules java.base/jdk.internal.access
  * @modules java.base/com.alibaba.wisp.engine:+open
  * @run main/othervm -XX:+EnableCoroutine -Dcom.alibaba.wisp.version=2 DaemonThreadGroupTest
 */
@@ -18,7 +18,7 @@
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicReference;
 
-import static jdk.testlibrary.Asserts.assertTrue;
+import static jdk.test.lib.Asserts.assertTrue;
 
 
 /**
diff --git a/test/jdk/com/alibaba/wisp/timer/OverflowTest.java b/test/jdk/com/alibaba/wisp/timer/OverflowTest.java
index 088c93debcf..8bec4d74c02 100644
--- a/test/jdk/com/alibaba/wisp/timer/OverflowTest.java
+++ b/test/jdk/com/alibaba/wisp/timer/OverflowTest.java
@@ -1,8 +1,8 @@
 /*
  * @test
- * @library /lib/testlibrary
+ * @library /test/lib
  * @summary Test timer implementation
- * @modules java.base/jdk.internal.misc
+ * @modules java.base/jdk.internal.access
  * @run main/othervm -XX:+EnableCoroutine OverflowTest
  * @run main/othervm -XX:+EnableCoroutine -Dcom.alibaba.wisp.version=2 OverflowTest
 */
@@ -16,7 +16,7 @@
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicReference;
 
-import static jdk.testlibrary.Asserts.assertTrue;
+import static jdk.test.lib.Asserts.assertTrue;
 
 
 /**
diff --git a/test/jdk/com/alibaba/wisp/timer/PriorityQueueSortTest.java b/test/jdk/com/alibaba/wisp/timer/PriorityQueueSortTest.java
index 0a4290cb922..5e2f1d8cf1f 100644
--- a/test/jdk/com/alibaba/wisp/timer/PriorityQueueSortTest.java
+++ b/test/jdk/com/alibaba/wisp/timer/PriorityQueueSortTest.java
@@ -1,6 +1,6 @@
 /*
  * @test
- * @library /lib/testlibrary
+ * @library /test/lib
  * @summary Test TimeOut.Queue's offer and remove function, make sure it's consistent with the behavior of the jdk's priority queue
  * @modules java.base/com.alibaba.wisp.engine:+open
  * @run main/othervm PriorityQueueSortTest
@@ -14,7 +14,7 @@
 import java.util.*;
 
 
-import static jdk.testlibrary.Asserts.assertTrue;
+import static jdk.test.lib.Asserts.assertTrue;
 
 public class PriorityQueueSortTest {
 
diff --git a/test/jdk/com/alibaba/wisp/timer/SleepRPCTest.java b/test/jdk/com/alibaba/wisp/timer/SleepRPCTest.java
index 85cabc40a85..93b48e07131 100644
--- a/test/jdk/com/alibaba/wisp/timer/SleepRPCTest.java
+++ b/test/jdk/com/alibaba/wisp/timer/SleepRPCTest.java
@@ -1,7 +1,7 @@
 /*
  * @test
  * @summary test use sleep in RPC senorina
- * @library /lib/testlibrary
+ * @library /test/lib
  * @run main/othervm -XX:+EnableCoroutine -XX:+UseWispMonitor -Dcom.alibaba.wisp.transparentWispSwitch=true  -Dcom.alibaba.wisp.enableThreadAsWisp=true SleepRPCTest
  * @run main/othervm -XX:+EnableCoroutine -XX:+UseWispMonitor -Dcom.alibaba.wisp.transparentWispSwitch=true  -Dcom.alibaba.wisp.enableThreadAsWisp=true -Dcom.alibaba.wisp.version=2 SleepRPCTest
  */
@@ -16,7 +16,7 @@
 import java.util.stream.Collectors;
 import java.util.stream.IntStream;
 
-import static jdk.testlibrary.Asserts.assertTrue;
+import static jdk.test.lib.Asserts.assertTrue;
 
 public class SleepRPCTest {
     public static void main(String[] args) {
diff --git a/test/jdk/com/alibaba/wisp/timer/SleepTest.java b/test/jdk/com/alibaba/wisp/timer/SleepTest.java
index 197e2e537b1..b6596888c90 100644
--- a/test/jdk/com/alibaba/wisp/timer/SleepTest.java
+++ b/test/jdk/com/alibaba/wisp/timer/SleepTest.java
@@ -1,7 +1,7 @@
 /*
  * @test
  * @summary test sleep
- * @library /lib/testlibrary
+ * @library /test/lib
  * @run main/othervm -XX:+EnableCoroutine -XX:+UseWispMonitor -Dcom.alibaba.wisp.transparentWispSwitch=true  -Dcom.alibaba.wisp.enableThreadAsWisp=true -Dcom.alibaba.wisp.version=2 SleepTest
  */
 
@@ -11,7 +11,7 @@
 import java.util.concurrent.TimeUnit;
 import java.util.stream.IntStream;
 
-import static jdk.testlibrary.Asserts.assertTrue;
+import static jdk.test.lib.Asserts.assertTrue;
 
 public class SleepTest {
     public static void main(String[] args) {
diff --git a/test/jdk/com/alibaba/wisp/timer/TimerTest.java b/test/jdk/com/alibaba/wisp/timer/TimerTest.java
index e221af92d07..14079946dc6 100644
--- a/test/jdk/com/alibaba/wisp/timer/TimerTest.java
+++ b/test/jdk/com/alibaba/wisp/timer/TimerTest.java
@@ -1,7 +1,7 @@
 /*
  * @test
  * @summary Test timer implement
- * @modules java.base/jdk.internal.misc
+ * @modules java.base/jdk.internal.access
  * @run main/othervm -XX:+EnableCoroutine TimerTest
  * @run main/othervm -XX:+EnableCoroutine -Dcom.alibaba.wisp.version=2 TimerTest
 */
diff --git a/test/jdk/com/alibaba/wisp2/AllThreadAsWispTest.java b/test/jdk/com/alibaba/wisp2/AllThreadAsWispTest.java
index 539bf39bf58..a5acdc1f90f 100644
--- a/test/jdk/com/alibaba/wisp2/AllThreadAsWispTest.java
+++ b/test/jdk/com/alibaba/wisp2/AllThreadAsWispTest.java
@@ -1,8 +1,8 @@
 /*
  * @test
- * @library /lib/testlibrary
+ * @library /test/lib
  * @summary convert all thread to wisp
- * @modules java.base/jdk.internal.misc
+ * @modules java.base/jdk.internal.access
  * @run main/othervm -XX:+EnableCoroutine  -Dcom.alibaba.wisp.transparentWispSwitch=true  -Dcom.alibaba.wisp.version=2 -XX:+UseWispMonitor -Dcom.alibaba.wisp.enableThreadAsWisp=true -Dcom.alibaba.wisp.allThreadAsWisp=true AllThreadAsWispTest
  */
 
@@ -14,7 +14,7 @@
 import java.util.concurrent.Executors;
 import java.util.concurrent.TimeUnit;
 
-import static jdk.testlibrary.Asserts.assertTrue;
+import static jdk.test.lib.Asserts.assertTrue;
 
 public class AllThreadAsWispTest {
     public static void main(String[] args) throws Exception {
diff --git a/test/jdk/com/alibaba/wisp2/CtxClassLoaderInheritanceTest.java b/test/jdk/com/alibaba/wisp2/CtxClassLoaderInheritanceTest.java
index e5a4b85a774..a25e37dd8a6 100644
--- a/test/jdk/com/alibaba/wisp2/CtxClassLoaderInheritanceTest.java
+++ b/test/jdk/com/alibaba/wisp2/CtxClassLoaderInheritanceTest.java
@@ -1,6 +1,6 @@
 /*
  * @test
- * @library /lib/testlibrary
+ * @library /test/lib
  * @summary test context ClassLoader inherit.
  * @run main/othervm -XX:+EnableCoroutine  -Dcom.alibaba.wisp.transparentWispSwitch=true  -Dcom.alibaba.wisp.version=2 -XX:+UseWispMonitor -Dcom.alibaba.wisp.enableThreadAsWisp=true -Dcom.alibaba.wisp.allThreadAsWisp=true CtxClassLoaderInheritanceTest
  */
@@ -8,7 +8,7 @@
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.TimeUnit;
 
-import static jdk.testlibrary.Asserts.assertEQ;
+import static jdk.test.lib.Asserts.assertEQ;
 
 public class CtxClassLoaderInheritanceTest {
     public static void main(String[] args) throws Exception {
diff --git a/test/jdk/com/alibaba/wisp2/DispatchTest.java b/test/jdk/com/alibaba/wisp2/DispatchTest.java
index 284672a7ca9..00c9c45e714 100644
--- a/test/jdk/com/alibaba/wisp2/DispatchTest.java
+++ b/test/jdk/com/alibaba/wisp2/DispatchTest.java
@@ -1,7 +1,7 @@
 /*
  * @test
  * @summary basic wisp2
- * @modules java.base/jdk.internal.misc
+ * @modules java.base/jdk.internal.access
  * @run main/othervm -XX:+EnableCoroutine  -Dcom.alibaba.wisp.transparentWispSwitch=true  -Dcom.alibaba.wisp.version=2 -XX:+UseWispMonitor  DispatchTest
  */
 
diff --git a/test/jdk/com/alibaba/wisp2/HandOffTest.java b/test/jdk/com/alibaba/wisp2/HandOffTest.java
index 8f2dcfd4fb3..b824d0261e7 100644
--- a/test/jdk/com/alibaba/wisp2/HandOffTest.java
+++ b/test/jdk/com/alibaba/wisp2/HandOffTest.java
@@ -1,6 +1,6 @@
 /*
  * @test
- * @library /lib/testlibrary
+ * @library /test/lib
  * @summary test long running or blocking syscall task could be retaken
  * @run main/othervm -Dcom.alibaba.wisp.carrierEngines=1 -XX:-UseBiasedLocking -XX:+EnableCoroutine -XX:+UseWispMonitor -Dcom.alibaba.wisp.transparentWispSwitch=true  -Dcom.alibaba.wisp.version=2 -Dcom.alibaba.wisp.enableHandOff=true -Dcom.alibaba.wisp.sysmonTickUs=100000 HandOffTest
  */
@@ -15,7 +15,7 @@
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
 
-import static jdk.testlibrary.Asserts.assertTrue;
+import static jdk.test.lib.Asserts.assertTrue;
 
 public class HandOffTest {
     public static void main(String[] args) throws Exception {
diff --git a/test/jdk/com/alibaba/wisp2/NioBlockingAcceptTest.java b/test/jdk/com/alibaba/wisp2/NioBlockingAcceptTest.java
index ff7389f8956..b881820b84b 100644
--- a/test/jdk/com/alibaba/wisp2/NioBlockingAcceptTest.java
+++ b/test/jdk/com/alibaba/wisp2/NioBlockingAcceptTest.java
@@ -1,6 +1,6 @@
 /*
  * @test
- * @library /lib/testlibrary
+ * @library /test/lib
  * @summary test nio blocking accept
  * @run main/othervm -Dcom.alibaba.wisp.carrierEngines=1 -XX:ActiveProcessorCount=1 -XX:+EnableCoroutine  -Dcom.alibaba.wisp.transparentWispSwitch=true  -Dcom.alibaba.wisp.version=2 -XX:+UseWispMonitor -Dcom.alibaba.wisp.enableThreadAsWisp=true -Dcom.alibaba.wisp.allThreadAsWisp=true NioBlockingAcceptTest
  */
@@ -13,7 +13,7 @@
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.TimeUnit;
 
-import static jdk.testlibrary.Asserts.assertTrue;
+import static jdk.test.lib.Asserts.assertTrue;
 
 public class NioBlockingAcceptTest {
     public static void main(String[] args) throws Exception {
diff --git a/test/jdk/com/alibaba/wisp2/ReuseWispTaskAfterThreadJoinTest.java b/test/jdk/com/alibaba/wisp2/ReuseWispTaskAfterThreadJoinTest.java
index 6c5db785e1b..cbc9c8b4e99 100644
--- a/test/jdk/com/alibaba/wisp2/ReuseWispTaskAfterThreadJoinTest.java
+++ b/test/jdk/com/alibaba/wisp2/ReuseWispTaskAfterThreadJoinTest.java
@@ -1,6 +1,6 @@
 /*
  * @test
- * @library /lib/testlibrary
+ * @library /test/lib
  * @summary test wisp task reusing after thread.join()
  * @run main/othervm  -XX:+EnableCoroutine  -Dcom.alibaba.wisp.transparentWispSwitch=true  -Dcom.alibaba.wisp.version=2 -XX:+UseWispMonitor -Dcom.alibaba.wisp.enableThreadAsWisp=true -Dcom.alibaba.wisp.allThreadAsWisp=true  ReuseWispTaskAfterThreadJoinTest
  */
diff --git a/test/jdk/com/alibaba/wisp2/ThreadJoinTest.java b/test/jdk/com/alibaba/wisp2/ThreadJoinTest.java
index d920775e936..1d9b0847106 100644
--- a/test/jdk/com/alibaba/wisp2/ThreadJoinTest.java
+++ b/test/jdk/com/alibaba/wisp2/ThreadJoinTest.java
@@ -1,6 +1,6 @@
 /*
  * @test
- * @library /lib/testlibrary
+ * @library /test/lib
  * @summary test thread.join()
  * @run main/othervm  -XX:+EnableCoroutine  -Dcom.alibaba.wisp.transparentWispSwitch=true  -Dcom.alibaba.wisp.version=2 -XX:+UseWispMonitor -Dcom.alibaba.wisp.enableThreadAsWisp=true -Dcom.alibaba.wisp.allThreadAsWisp=true ThreadJoinTest
  */
diff --git a/test/jdk/com/alibaba/wisp2/Wisp2ShutdownTest.java b/test/jdk/com/alibaba/wisp2/Wisp2ShutdownTest.java
index 5eabd763686..de9091d156e 100644
--- a/test/jdk/com/alibaba/wisp2/Wisp2ShutdownTest.java
+++ b/test/jdk/com/alibaba/wisp2/Wisp2ShutdownTest.java
@@ -2,7 +2,7 @@
  * @test
  * @summary Wisp2ShutdownTest
  * @modules java.base/com.alibaba.wisp.engine:+open
- * @library /lib/testlibrary
+ * @library /test/lib
  * @run main/othervm -XX:+EnableCoroutine -XX:+UseWispMonitor -Dcom.alibaba.wisp.transparentWispSwitch=true -Dcom.alibaba.wisp.version=2 Wisp2ShutdownTest
  */
 
@@ -12,7 +12,7 @@
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicInteger;
 
-import static jdk.testlibrary.Asserts.assertEQ;
+import static jdk.test.lib.Asserts.assertEQ;
 
 public class Wisp2ShutdownTest {
     public static void main(String[] args) throws Exception {
@@ -39,6 +39,7 @@ public Thread newThread(Runnable r) {
                     n.decrementAndGet();
                 }
             });
+
         }
 
         while (n.get() != 888) {
@@ -49,7 +50,7 @@ public Thread newThread(Runnable r) {
 
         g.shutdown();
 
-        g.awaitTermination(1, TimeUnit.SECONDS);
+        g.awaitTermination(5, TimeUnit.SECONDS);
 
         System.out.println(System.currentTimeMillis() - start + "ms");
 
diff --git a/test/jdk/com/alibaba/wisp2/Wisp2WaitNotifyTest.java b/test/jdk/com/alibaba/wisp2/Wisp2WaitNotifyTest.java
index ed1e85db07b..42ab7b54624 100644
--- a/test/jdk/com/alibaba/wisp2/Wisp2WaitNotifyTest.java
+++ b/test/jdk/com/alibaba/wisp2/Wisp2WaitNotifyTest.java
@@ -1,6 +1,6 @@
 /*
  * @test
- * @library /lib/testlibrary
+ * @library /test/lib
  * @summary Test Object.wait/notify with coroutine in wisp2
  * @run main/othervm  -XX:-UseBiasedLocking -XX:+EnableCoroutine -XX:+UseWispMonitor -Dcom.alibaba.wisp.transparentWispSwitch=true -Dcom.alibaba.wisp.version=2 Wisp2WaitNotifyTest
  */
@@ -10,7 +10,7 @@
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.atomic.AtomicInteger;
 
-import static jdk.testlibrary.Asserts.assertEQ;
+import static jdk.test.lib.Asserts.assertEQ;
 
 public class Wisp2WaitNotifyTest {
 
diff --git a/test/jdk/com/alibaba/wisp2/Wisp2WorkStealTest.java b/test/jdk/com/alibaba/wisp2/Wisp2WorkStealTest.java
index f9dc6938047..f95b0cc0c79 100644
--- a/test/jdk/com/alibaba/wisp2/Wisp2WorkStealTest.java
+++ b/test/jdk/com/alibaba/wisp2/Wisp2WorkStealTest.java
@@ -1,8 +1,8 @@
 /*
  * @test
- * @library /lib/testlibrary
+ * @library /test/lib
  * @summary verification of work stealing really happened
- * @modules java.base/jdk.internal.misc
+ * @modules java.base/jdk.internal.access
  * @run main/othervm -XX:+UseWisp2 Wisp2WorkStealTest
  */
 
@@ -11,7 +11,7 @@
 
 import java.util.concurrent.atomic.AtomicReference;
 
-import static jdk.testlibrary.Asserts.assertNE;
+import static jdk.test.lib.Asserts.assertNE;
 
 public class Wisp2WorkStealTest {
     public static void main(String[] args) {
diff --git a/test/jdk/com/alibaba/wisp2/Wisp2YieldTest.java b/test/jdk/com/alibaba/wisp2/Wisp2YieldTest.java
index c5ade47a334..eb05604e7be 100644
--- a/test/jdk/com/alibaba/wisp2/Wisp2YieldTest.java
+++ b/test/jdk/com/alibaba/wisp2/Wisp2YieldTest.java
@@ -1,13 +1,13 @@
 /*
  * @test
- * @library /lib/testlibrary
+ * @library /test/lib
  * @summary Test yield in wisp2
  * @run main/othervm  -XX:-UseBiasedLocking -XX:+EnableCoroutine -XX:+UseWispMonitor -Dcom.alibaba.wisp.transparentWispSwitch=true -Dcom.alibaba.wisp.version=2 -Dcom.alibaba.wisp.workerEngines=1 Wisp2YieldTest
  */
 
 import com.alibaba.wisp.engine.WispEngine;
 
-import static jdk.testlibrary.Asserts.assertTrue;
+import static jdk.test.lib.Asserts.assertTrue;
 
 
 public class Wisp2YieldTest {
diff --git a/test/jdk/com/alibaba/wisp2/bug/ConcurrentThreadJoinTest.java b/test/jdk/com/alibaba/wisp2/bug/ConcurrentThreadJoinTest.java
index cba11d95bbf..4cdfb9dd1fb 100644
--- a/test/jdk/com/alibaba/wisp2/bug/ConcurrentThreadJoinTest.java
+++ b/test/jdk/com/alibaba/wisp2/bug/ConcurrentThreadJoinTest.java
@@ -1,11 +1,11 @@
 /*
  * @test
- * @library /lib/testlibrary
+ * @library /test/lib
  * @summary ensure thread.isAlive() is false after thread.join()
  * @run main/othervm -XX:+UseWisp2 ConcurrentThreadJoinTest
  */
 
-import static jdk.testlibrary.Asserts.assertFalse;
+import static jdk.test.lib.Asserts.assertFalse;
 
 public class ConcurrentThreadJoinTest {
     public static void main(String[] args) throws Exception {
diff --git a/test/jdk/com/alibaba/wisp2/bug/DisableStealBugTest.java b/test/jdk/com/alibaba/wisp2/bug/DisableStealBugTest.java
index d5e4ecc6a84..95eeba663d2 100644
--- a/test/jdk/com/alibaba/wisp2/bug/DisableStealBugTest.java
+++ b/test/jdk/com/alibaba/wisp2/bug/DisableStealBugTest.java
@@ -1,7 +1,7 @@
 /*
  * @test
- * @library /lib/testlibrary
- * @modules java.base/jdk.internal.misc
+ * @library /test/lib
+ * @modules java.base/jdk.internal.access
  * @modules java.base/com.alibaba.wisp.engine:+open
  * @summary test bug of update stealEnable fail
  * @run main/othervm -XX:+UseWisp2 -XX:-EnableSteal DisableStealBugTest
@@ -14,7 +14,7 @@
 import java.lang.reflect.Field;
 import java.util.concurrent.atomic.AtomicReference;
 
-import static jdk.testlibrary.Asserts.assertTrue;
+import static jdk.test.lib.Asserts.assertTrue;
 
 public class DisableStealBugTest {
     public static void main(String[] args) throws Exception {
diff --git a/test/jdk/com/alibaba/wisp2/bug/Wisp2ThreadObjLeakInThreadGroupTest.java b/test/jdk/com/alibaba/wisp2/bug/Wisp2ThreadObjLeakInThreadGroupTest.java
index 43532427ddd..05646d161c8 100644
--- a/test/jdk/com/alibaba/wisp2/bug/Wisp2ThreadObjLeakInThreadGroupTest.java
+++ b/test/jdk/com/alibaba/wisp2/bug/Wisp2ThreadObjLeakInThreadGroupTest.java
@@ -1,13 +1,13 @@
 /*
  * @test
- * @library /lib/testlibrary
+ * @library /test/lib
  * @summary test bug fix of thread object leak in thread group
  * @run main/othervm -XX:+UseWisp2 Wisp2ThreadObjLeakInThreadGroupTest
  */
 
 import java.util.concurrent.CountDownLatch;
 
-import static jdk.testlibrary.Asserts.assertEQ;
+import static jdk.test.lib.Asserts.assertEQ;
 
 public class Wisp2ThreadObjLeakInThreadGroupTest {
     public static void main(String[] args) throws Exception {
diff --git a/test/jdk/java/beans/XMLDecoder/8028054/Task.java b/test/jdk/java/beans/XMLDecoder/8028054/Task.java
index 1d55fa8e358..0f96be3d5bb 100644
--- a/test/jdk/java/beans/XMLDecoder/8028054/Task.java
+++ b/test/jdk/java/beans/XMLDecoder/8028054/Task.java
@@ -130,6 +130,11 @@ static List> getClasses(int count) throws Exception {
                 .map(s -> s.substring(s.indexOf("java")))
                 .collect(Collectors.toList());
 
+        // If enclude java.dyn.CoroutineSupport, it will cause the following error:
+        // java.lang.UnsatisfiedLinkError: 'void java.dyn.CoroutineSupport.registerNatives()'
+        // Let any brilliant guy to fix it.
+        fileNames.remove("java.dyn.CoroutineSupport");
+
         for (String name : fileNames) {
             classes.add(Class.forName(name));
             if (count == classes.size()) {
diff --git a/test/jdk/java/dyn/BasicStealTest.java b/test/jdk/java/dyn/BasicStealTest.java
index c878f3e7a75..7a3a00c5d08 100644
--- a/test/jdk/java/dyn/BasicStealTest.java
+++ b/test/jdk/java/dyn/BasicStealTest.java
@@ -1,7 +1,7 @@
 /*
  * @test
  * @summary test basic coroutine steal mechanism
- * @library /lib/testlibrary
+ * @library /test/lib
  * @run main/othervm -XX:+EnableCoroutine BasicStealTest
  */
 
@@ -9,8 +9,8 @@
 import java.util.concurrent.atomic.AtomicReference;
 import java.util.function.Consumer;
 
-import static jdk.testlibrary.Asserts.assertEQ;
-import static jdk.testlibrary.Asserts.assertTrue;
+import static jdk.test.lib.Asserts.assertEQ;
+import static jdk.test.lib.Asserts.assertTrue;
 
 public class BasicStealTest {
     public static void main(String[] args) {
diff --git a/test/jdk/jdk/jfr/event/runtime/TestSystemPropertyEvent.java b/test/jdk/jdk/jfr/event/runtime/TestSystemPropertyEvent.java
index 9cb506c45cb..b1bc3ad21c5 100644
--- a/test/jdk/jdk/jfr/event/runtime/TestSystemPropertyEvent.java
+++ b/test/jdk/jdk/jfr/event/runtime/TestSystemPropertyEvent.java
@@ -105,6 +105,11 @@ private static Map createInitialSystemProperties() {
             result.put(key, System.getProperty(key));
             System.out.println("initialProp: " + key);
         }
+        {
+            // Add a property that is not set at JVM start.
+            String key = "com.alibaba.coroutine.enableCoroutine";
+            result.put(key, System.getProperty(key));
+        }
         return result;
     }
 }