feat(op_status_lock): add a command to unlock meta_op_status manually

include/dsn/dist/replication/replication.codes.h

@@ -124,6 +124,7 @@ MAKE_EVENT_CODE_RPC(RPC_CM_START_BACKUP_APP, TASK_PRIORITY_COMMON)
 MAKE_EVENT_CODE_RPC(RPC_CM_QUERY_BACKUP_STATUS, TASK_PRIORITY_COMMON)
 MAKE_EVENT_CODE_RPC(RPC_CM_START_MANUAL_COMPACT, TASK_PRIORITY_COMMON)
 MAKE_EVENT_CODE_RPC(RPC_CM_QUERY_MANUAL_COMPACT_STATUS, TASK_PRIORITY_COMMON)
+MAKE_EVENT_CODE_RPC(RPC_CM_UNLOCK_META_OP_STATUS, TASK_PRIORITY_COMMON)
 #undef CURRENT_THREAD_POOL
 
 #define CURRENT_THREAD_POOL THREAD_POOL_META_STATE

include/dsn/dist/replication/replication_ddl_client.h

@@ -228,6 +228,8 @@ class replication_ddl_client
     error_with<query_app_manual_compact_response>
     query_app_manual_compact(const std::string &app_name);
 
+    dsn::error_code unlock_meta_op_status();
+
 private:
     bool static valid_app_char(int c);
 

src/client/replication_ddl_client.cpp

@@ -1712,5 +1712,23 @@ replication_ddl_client::query_app_manual_compact(const std::string &app_name)
         query_manual_compact_rpc(std::move(req), RPC_CM_QUERY_MANUAL_COMPACT_STATUS));
 }
 
+dsn::error_code replication_ddl_client::unlock_meta_op_status()
+{
+    std::shared_ptr<unlock_meta_op_status_request> req =
+        std::make_shared<unlock_meta_op_status_request>();
+
+    auto resp_task = request_meta<unlock_meta_op_status_request>(RPC_CM_UNLOCK_META_OP_STATUS, req);
+
+    resp_task->wait();
+    if (resp_task->error() != ERR_OK) {
+        return resp_task->error();
+    }
+
+    unlock_meta_op_status_response resp;
+    ::dsn::unmarshall(resp_task->get_response(), resp);
+    std::cout << "unlock_meta_op_status: " << resp.err.to_string() << std::endl;
+    return resp.err;
+}
+
 } // namespace replication
 } // namespace dsn

src/common/meta_admin.thrift

@@ -136,6 +136,15 @@ struct configuration_recovery_response
     2:string hint_message;
 }
 
+struct unlock_meta_op_status_request
+{
+}
+
+struct unlock_meta_op_status_response
+{
+    1:dsn.error_code err;
+}
+
 /////////////////// Tables Management ////////////////////
 
 struct create_app_options

src/meta/meta_rpc_types.h

@@ -56,6 +56,8 @@ typedef rpc_holder<configuration_modify_backup_policy_request,
 typedef rpc_holder<start_backup_app_request, start_backup_app_response> start_backup_app_rpc;
 typedef rpc_holder<query_backup_status_request, query_backup_status_response>
     query_backup_status_rpc;
+typedef rpc_holder<unlock_meta_op_status_request, unlock_meta_op_status_response>
+    unlock_meta_op_status_rpc;
 
 } // namespace replication
 } // namespace dsn

src/meta/meta_service.cpp

@@ -276,9 +276,11 @@ bool meta_service::try_lock_meta_op_status(meta_op_status op_status)
     return true;
 }
 
-void meta_service::unlock_meta_op_status()
+void meta_service::unlock_meta_op_status(bool is_manual)
 {
-    ddebug_f("UNLOCK meta op status from {}", enum_to_string(_meta_op_status.load()));
+    ddebug_f("UNLOCK meta op status from {}, is_manual {}",
+             enum_to_string(_meta_op_status.load()),
+             is_manual);
     _meta_op_status.store(meta_op_status::FREE);
 }
 
@@ -546,6 +548,9 @@ void meta_service::register_rpc_handlers()
     register_rpc_handler_with_rpc_holder(RPC_CM_QUERY_MANUAL_COMPACT_STATUS,
                                          "query_manual_compact_status",
                                          &meta_service::on_query_manual_compact_status);
+    register_rpc_handler_with_rpc_holder(RPC_CM_UNLOCK_META_OP_STATUS,
+                                         "unlock_meta_op_status",
+                                         &meta_service::on_unlock_meta_op_status);
 }
 
 int meta_service::check_leader(dsn::message_ex *req, dsn::rpc_address *forward_address)
@@ -693,9 +698,20 @@ void meta_service::on_query_cluster_info(configuration_cluster_info_rpc rpc)
     response.values.push_back(fmt::format("{:.{}f}", total_stddev, 2));
     response.keys.push_back("cluster_name");
     response.values.push_back(get_current_cluster_name());
+    response.keys.push_back("meta_op_status");
+    response.values.push_back(enum_to_string(get_op_status()));
     response.err = dsn::ERR_OK;
 }
 
+void meta_service::on_unlock_meta_op_status(unlock_meta_op_status_rpc rpc)
+{
+    if (!check_status(rpc)) {
+        return;
+    }
+    unlock_meta_op_status(true);
+    rpc.response().err = ERR_OK;
+}
+
 // client => meta server
 void meta_service::on_query_configuration_by_index(configuration_query_by_index_rpc rpc)
 {

src/meta/meta_service.h

@@ -162,7 +162,7 @@ class meta_service : public serverlet<meta_service>
     size_t get_alive_node_count() const;
 
     bool try_lock_meta_op_status(meta_op_status op_status);
-    void unlock_meta_op_status();
+    void unlock_meta_op_status(bool is_manual = false);
     meta_op_status get_op_status() const { return _meta_op_status.load(); }
 
 private:
@@ -195,6 +195,7 @@ class meta_service : public serverlet<meta_service>
 
     // cluster info
     void on_query_cluster_info(configuration_cluster_info_rpc rpc);
+    void on_unlock_meta_op_status(unlock_meta_op_status_rpc rpc);
 
     // meta control
     void on_control_meta_level(configuration_meta_control_rpc rpc);