[Runtime] Determine tracer or replayer for QuickStart

Summary:
With -Xquickstart, there are three roles for the java processes:
1. tracer
first create the LOCK file, and mark the process as tracer.
create the metadata file after the process has started,
2. replayer
If the metadata file is found, mark the process as replayer
3. normal
If there is no metadata file, and failed to create the LOCK file,
the process will run without quickstart feature.

-Xquickstart:destroy
delete the directory of the cache file, and exit.

More detail in https://yuque.antfin-inc.com/aone355606/uzt0dx/tylp9b

Test Plan:
test/jdk/com/alibaba/quickstart/TestDeterminingTracerOrReplayer.java
test/jdk/com/alibaba/quickstart/TestParallelProcessRun.java
test/jdk/com/alibaba/quickstart/TestRemoveDirectory.java

src/hotspot/share/prims/whitebox.cpp

@@ -80,6 +80,7 @@
 #include "runtime/jniHandles.inline.hpp"
 #include "runtime/os.hpp"
 #include "runtime/stackFrameStream.inline.hpp"
+#include "runtime/quickStart.hpp"
 #include "runtime/sweeper.hpp"
 #include "runtime/synchronizer.hpp"
 #include "runtime/thread.hpp"
@@ -1370,6 +1371,14 @@ WB_ENTRY(jboolean, WB_IsInStringTable(JNIEnv* env, jobject o, jstring javaString
   return (StringTable::lookup(name, len) != NULL);
 WB_END
 
+WB_ENTRY(void, WB_RemoveDirectory(JNIEnv* env, jobject o, jstring javaString))
+  char* dir = NULL;
+  if (javaString != NULL) {
+    dir = java_lang_String::as_utf8_string(JNIHandles::resolve_non_null(javaString));
+    QuickStart::remove_dir(dir);
+  }
+WB_END
+
 WB_ENTRY(void, WB_FullGC(JNIEnv* env, jobject o))
   Universe::heap()->soft_ref_policy()->set_should_clear_all_soft_refs(true);
   Universe::heap()->collect(GCCause::_wb_full_gc);
@@ -2494,6 +2503,7 @@ static JNINativeMethod methods[] = {
   {CC"getStringVMFlag",    CC"(Ljava/lang/String;)Ljava/lang/String;",
                                                       (void*)&WB_GetStringVMFlag},
   {CC"isInStringTable",    CC"(Ljava/lang/String;)Z", (void*)&WB_IsInStringTable  },
+  {CC"removeDirectory",    CC"(Ljava/lang/String;)V", (void*)&WB_RemoveDirectory },
   {CC"fullGC",   CC"()V",                             (void*)&WB_FullGC },
   {CC"youngGC",  CC"()V",                             (void*)&WB_YoungGC },
   {CC"readReservedMemory", CC"()V",                   (void*)&WB_ReadReservedMemory },

src/hotspot/share/runtime/quickStart.cpp

@@ -1,22 +1,26 @@
 #include "precompiled.hpp"
+#include <limits.h>
 #include <stdlib.h>
 #include <strings.h>
 #include "classfile/vmSymbols.hpp"
 #include "runtime/java.hpp"
 #include "runtime/quickStart.hpp"
 #include "utilities/defaultStream.hpp"
 
-bool QuickStart::_is_enabled = false;
-bool QuickStart::_is_tracer = false;
-bool QuickStart::_is_replayer = false;
 bool QuickStart::_is_starting = true;
+bool QuickStart::_is_enabled = false;
 bool QuickStart::_verbose = false;
 bool QuickStart::_print_stat_enabled = false;
 bool QuickStart::_need_destroy = false;
 bool QuickStart::_need_finish_check = true;
 
+QuickStart::QuickStartRole QuickStart::_role = QuickStart::Normal;
+
 const char* QuickStart::_cache_path = NULL;
 const char* QuickStart::_image_env = NULL;
+const char* QuickStart::_lock_path = NULL;
+const char* QuickStart::_temp_metadata_file_path = NULL;
+const char* QuickStart::_metadata_file_path = NULL;
 const char* QuickStart::_opt_name[] = {
 #define OPT_TAG(name) #name,
   OPT_TAG_LIST
@@ -29,8 +33,18 @@ bool QuickStart::_opt_enabled[] = {
 #undef OPT_TAG
 };
 
-#define DEFAULT_DIRECTORY "sharedcache"
-#define MAX_PATH (2 * K)
+FILE *QuickStart::_METADATA_FILE = NULL;
+FILE *QuickStart::_TEMP_METADATA_FILE = NULL;
+
+int QuickStart::_lock_file_fd = 0;
+
+#define DEFAULT_SHARED_DIRECTORY      "alibaba.quickstart.sharedcache"
+#define METADATA_FILE                 "metadata"
+// metadata file stores some information for invalid check, and generate it after the startup is complete.
+// Before the content of the file is completely generated, use the file name metadata.tmp temporarily.
+// After the startup is complete, rename metadata.tmp to metadata.
+#define TEMP_METADATA_FILE            "metadata.tmp"
+#define LOCK_FILE                     "LOCK"
 
 bool QuickStart::parse_command_line_arguments(const char* options) {
   _is_enabled = true;
@@ -59,7 +73,7 @@ bool QuickStart::parse_command_line_arguments(const char* options) {
       _verbose = true;
     } else if (match_option(cur, "printStat", &tail)) {
       _print_stat_enabled = true;
-    } else if (match_option(cur, "destory", &tail)) {
+    } else if (match_option(cur, "destroy", &tail)) {
       _need_destroy = true;
     } else if (match_option(cur, "path=", &tail)) {
       _cache_path = os::strdup_check_oom(tail, mtArguments);
@@ -134,41 +148,202 @@ void QuickStart::initialize(TRAPS) {
 void QuickStart::post_process_arguments() {
   // Prepare environment
   calculate_cache_path();
+  // destroy the cache directory
+  destroy_cache_folder();
   // Determine the role
-  determine_tracer_or_replayer();
-  // check the consistency
-  check_consistency();
+  if (!determine_tracer_or_replayer()) {
+    _role = Normal;
+    return;
+  }
   // Process argument for each optimization
   process_argument_for_optimaztion();
+
+  if (_role == Tracer) {
+    // Temporarily put here to ensure the integrity of the test
+    generate_metadata_file();
+  }
 }
 
-void QuickStart::check_consistency() {
-  // check the consistency
+bool QuickStart::check_integrity() {
+  // check the integrity
+  return true;
 }
 
 void QuickStart::calculate_cache_path() {
   if (_cache_path != NULL) {
-    if (_verbose) tty->print_cr("[QuickStart] cache path is set from -Xquickstart:path=%s", _cache_path);
+    log("cache path is set from -Xquickstart:path=%s", _cache_path);
     return;
   }
 
   const char *buffer = ::getenv("QUICKSTART_CACHE_PATH");
   if (buffer != NULL && (_cache_path = os::strdup_check_oom(buffer)) != NULL) {
-    if (_verbose) tty->print_cr("[QuickStart] cache path is set from env with %s", _cache_path);
+    log("cache path is set from env with %s", _cache_path);
     return;
   }
   const char* home = ::getenv("HOME");
-  char buf[MAX_PATH];
-  jio_snprintf(buf, MAX_PATH, "%s%s%s", home, os::file_separator, DEFAULT_DIRECTORY);
+  char buf[PATH_MAX];
+  jio_snprintf(buf, PATH_MAX, "%s%s%s", home, os::file_separator(), DEFAULT_SHARED_DIRECTORY);
   _cache_path = os::strdup_check_oom(buf);
-  if (_verbose) tty->print_cr("[QuickStart] cache path is set as default with %s", _cache_path);
+  log("cache path is set as default with %s", _cache_path);
+}
+
+void QuickStart::destroy_cache_folder() {
+  if (_need_destroy && _cache_path != NULL) {
+    if (remove_dir(_cache_path) < 0) {
+      log("failed to destory the cache folder: %s", _cache_path);
+    } else {
+      log("destory the cache folder: %s", _cache_path);
+    }
+    vm_exit(0);
+  }
 }
 
 void QuickStart::process_argument_for_optimaztion() {
-  if (_is_replayer) {
-  } else if (_is_tracer) {
+  switch(_role) {
+    case Replayer:
+      break;
+    case Tracer:
+      break;
+    default:
+      break;
   }
 }
 
-void QuickStart::determine_tracer_or_replayer() {
+bool QuickStart::determine_tracer_or_replayer() {
+  struct stat st;
+  char buf[PATH_MAX];
+  int ret = os::stat(_cache_path, &st);
+  if (ret != 0) {
+    ret = ::mkdir(_cache_path, S_IRWXU | S_IRWXG | S_IROTH | S_IXOTH);
+    if (ret != 0) {
+      log("Could not mkdir [%s] because [%s]", _cache_path, os::strerror(errno));
+      return false;
+    }
+  } else if (!S_ISDIR(st.st_mode)) {
+    log("Cache path [%s] is not a directory, "
+        "please use -Xquickstart:path=<path> or environment variable "
+        "QUICKSTART_CACHE_PATH to specify.\n",
+        _cache_path);
+    return false;
+  }
+
+  // check whether the metadata file exists.
+  jio_snprintf(buf, PATH_MAX, "%s%s%s", _cache_path, os::file_separator(), METADATA_FILE);
+  _metadata_file_path = os::strdup_check_oom(buf, mtArguments);
+  ret = os::stat(_metadata_file_path, &st);
+  if (ret < 0 && errno == ENOENT) {
+    // Create a LOCK file
+    jio_snprintf(buf, PATH_MAX, "%s%s%s", _cache_path, os::file_separator(), LOCK_FILE);
+    _lock_path = os::strdup_check_oom(buf, mtArguments);
+    // if the lock exists, it returns -1.
+    _lock_file_fd = os::create_binary_file(_lock_path, false);
+    if (_lock_file_fd == -1) {
+      log("Fail to create LOCK file");
+      return false;
+    }
+    jio_snprintf(buf, PATH_MAX, "%s%s%s", _cache_path, os::file_separator(), TEMP_METADATA_FILE);
+    _temp_metadata_file_path = os::strdup_check_oom(buf, mtArguments);
+    ret = os::stat(buf, &st);
+    if (ret == 0) {
+      // error: A file exists, determine failed. Maybe this is a user's file.
+      jio_fprintf(defaultStream::error_stream(), "[QuickStart] The %s file exists\n", TEMP_METADATA_FILE);
+      return false;
+    }
+    _TEMP_METADATA_FILE = os::fopen(buf, "w");
+    if (!_TEMP_METADATA_FILE) {
+      jio_fprintf(defaultStream::error_stream(), "[QuickStart] Failed to create %s file\n", TEMP_METADATA_FILE);
+      return false;
+    }
+    _role = Tracer;
+    log("Running as tracer");
+    return true;
+  } else if (ret == 0 && check_integrity()) {
+    _role = Replayer;
+    log("Running as replayer");
+    return true;
+  }
+  return false;
+}
+
+void QuickStart::generate_metadata_file() {
+  // mv metadata to metadata.tmp
+  ::fclose(_TEMP_METADATA_FILE);
+  int ret = ::rename(_temp_metadata_file_path, _metadata_file_path);
+  if (ret != 0) {
+    jio_fprintf(defaultStream::error_stream(),
+                "[QuickStart] Could not mv [%s] to [%s] because [%s]\n",
+                TEMP_METADATA_FILE,
+                METADATA_FILE,
+                os::strerror(errno));
+  }
+
+  // remove lock file
+  ret = ::close(_lock_file_fd);
+  if (ret != 0) {
+    jio_fprintf(defaultStream::error_stream(),
+                "[QuickStart] Could not close [%s] because [%s]\n",
+                LOCK_FILE,
+                os::strerror(errno));
+  }
+
+  ret = ::remove(_lock_path);
+  if (ret != 0) {
+    jio_fprintf(defaultStream::error_stream(),
+                "[QuickStart] Could not delete [%s] because [%s]\n",
+                LOCK_FILE,
+                os::strerror(errno));
+  }
+}
+
+void QuickStart::log(const char* msg, ...) {
+  if (_verbose) {
+    va_list ap;
+    va_start(ap, msg);
+    tty->print("[QuickStart(%d)] ", os::current_process_id());
+    tty->vprint(msg, ap);
+    tty->print_cr("");
+    va_end(ap);
+  }
+}
+
+int QuickStart::remove_dir(const char* dir) {
+  char cur_dir[] = ".";
+  char up_dir[]  = "..";
+  char dir_name[PATH_MAX];
+  DIR *dirp = NULL;
+  struct dirent *dp;
+  struct stat dir_stat;
+
+  int ret = os::stat(dir, &dir_stat);
+  if (ret < 0) {
+    jio_fprintf(defaultStream::error_stream(),
+                "[QuickStart] Fail to get the stat for directory %s\n",
+                dir);
+    return ret;
+  }
+
+  if (S_ISREG(dir_stat.st_mode)) {
+    ret = ::remove(dir);
+  } else if (S_ISDIR(dir_stat.st_mode)) {
+    dirp = os::opendir(dir);
+    while ((dp = os::readdir(dirp)) != NULL) {
+      if ((strcmp(cur_dir, dp->d_name) == 0) || (strcmp(up_dir, dp->d_name) == 0)) {
+        continue;
+      }
+      jio_snprintf(dir_name, PATH_MAX, "%s%s%s", dir, os::file_separator(), dp->d_name);
+      ret = remove_dir(dir_name);
+      if (ret != 0) {
+        break;
+      }
+    }
+
+    os::closedir(dirp);
+    if (ret != 0) {
+      return -1;
+    }
+    ret = ::rmdir(dir);
+  } else {
+    jio_fprintf(defaultStream::error_stream(), "[QuickStart] unknow file type\n");
+  }
+  return ret;
 }

src/hotspot/share/runtime/quickStart.hpp

@@ -7,8 +7,6 @@
 #define OPT_TAG_LIST_EXT
 
 #define OPT_TAG_LIST \
-  OPT_TAG(cds) \
-  OPT_TAG(eagerappcds) \
   OPT_TAG_LIST_EXT
 
 class QuickStart : AllStatic {
@@ -20,23 +18,40 @@ class QuickStart : AllStatic {
     Count
   };
 
+  typedef enum {
+    Normal,
+    Tracer,
+    Replayer
+  } QuickStartRole;
+
   ~QuickStart();
   static const char* cache_path()       { return _cache_path; }
   static bool is_enabled()              { return _is_enabled; }
   static bool parse_command_line_arguments(const char* opts = NULL);
   static void post_process_arguments();
   static void initialize(TRAPS);
-  static bool is_producer()             { return _is_enabled && _is_tracer; }
-  static bool is_consumer()             { return _is_enabled && _is_replayer; }
-  static bool is_starting()             { return _is_enabled && _is_starting; }
+  static bool is_tracer()               { return _role == Tracer; }
+  static bool is_replayer()             { return _role == Replayer; }
+  static bool is_starting()             { return is_enabled() && _is_starting; }
+
+  static int remove_dir(const char* dir);
 
 private:
   static const char* _cache_path;
   static const char* _image_env;
-  static bool _is_enabled;
-  static bool _is_tracer;
-  static bool _is_replayer;
+  static const char* _lock_path;
+  static const char* _temp_metadata_file_path;
+  static const char* _metadata_file_path;
+
+  static FILE *_METADATA_FILE;
+  static FILE *_TEMP_METADATA_FILE;
+
+  static int _lock_file_fd;
+
+  static QuickStartRole _role;
+
   static bool _is_starting;
+  static bool _is_enabled;
   static bool _verbose;
   static bool _print_stat_enabled;
   static bool _need_destroy;
@@ -45,12 +60,15 @@ class QuickStart : AllStatic {
   static bool _opt_enabled[];
 
   static bool set_optimization(const char* option, bool enabled);
-  static void determine_tracer_or_replayer();
+  static bool determine_tracer_or_replayer();
   static void calculate_cache_path();
+  static void destroy_cache_folder();
   static void process_argument_for_optimaztion();
-  static void check_consistency();
+  static bool check_integrity();
+  static void generate_metadata_file();
   static bool match_option(const char* option, const char* name, const char** tail);
   static void print_command_line_help(outputStream* out);
+  static void log(const char* msg, ...) ATTRIBUTE_PRINTF(1, 2);
 };
 
 #endif