refactor: add DatabaseMeta.gc_in_progress

src/meta/api/src/name_id_value_api.rs

@@ -449,6 +449,7 @@ mod tests {
             updated_on: Default::default(),
             comment: "".to_string(),
             drop_on: None,
+            gc_in_progress: false,
         };
 
         let v = db_meta(1).to_pb()?.encode_to_vec();

src/meta/app/src/schema/database.rs

@@ -69,8 +69,29 @@ pub struct DatabaseMeta {
     pub updated_on: DateTime<Utc>,
     pub comment: String,
 
-    // if used in CreateDatabaseReq, this field MUST set to None.
+    /// if used in CreateDatabaseReq, this field MUST set to None.
     pub drop_on: Option<DateTime<Utc>>,
+
+    /// Indicates whether garbage collection is currently in progress for this dropped database.
+    ///
+    /// If it is in progress, the database should not be un-dropped, because the data may be incomplete.
+    ///
+    /// ```text
+    /// normal <----.
+    ///   |         |
+    ///   | drop()  | undrop()
+    ///   v         |
+    /// dropped ----'
+    ///   |
+    ///   | gc()
+    ///   v
+    /// gc_in_progress=True
+    ///   |
+    ///   | purge data from meta-service
+    ///   v
+    /// completed removed
+    /// ```
+    pub gc_in_progress: bool,
 }
 
 impl Default for DatabaseMeta {
@@ -83,6 +104,7 @@ impl Default for DatabaseMeta {
             updated_on: Utc::now(),
             comment: "".to_string(),
             drop_on: None,
+            gc_in_progress: false,
         }
     }
 }

src/meta/proto-conv/src/database_from_to_protobuf_impl.rs

@@ -44,6 +44,7 @@ impl FromToProto for mt::DatabaseMeta {
                 Some(drop_on) => Some(DateTime::<Utc>::from_pb(drop_on)?),
                 None => None,
             },
+            gc_in_progress: p.gc_in_progress,
             comment: p.comment,
         };
         Ok(v)
@@ -62,6 +63,7 @@ impl FromToProto for mt::DatabaseMeta {
                 Some(drop_on) => Some(drop_on.to_pb()?),
                 None => None,
             },
+            gc_in_progress: self.gc_in_progress,
             comment: self.comment.clone(),
             shared_by: vec![],
             from_share: None,

src/meta/proto-conv/src/util.rs

@@ -139,6 +139,7 @@ const META_CHANGE_LOG: &[(u64, &str)] = &[
     (107, "2024-08-09: Add: datatype.proto/DataType Geography type"),
     (108, "2024-08-29: Add: procedure.proto: ProcedureMeta and ProcedureIdentity"),
     (109, "2024-08-29: Refactor: ProcedureMeta add arg_names"),
+    (110, "2024-09-18: Add: database.proto: DatabaseMeta.gc_in_progress"),
     // Dear developer:
     //      If you're gonna add a new metadata version, you'll have to add a test for it.
     //      You could just copy an existing test file(e.g., `../tests/it/v024_table_meta.rs`)

src/meta/proto-conv/tests/it/main.rs

@@ -107,3 +107,4 @@ mod v106_query_token;
 mod v107_geography_datatype;
 mod v108_procedure;
 mod v109_procedure_with_args;
+mod v110_database_meta_gc_in_progress;

src/meta/proto-conv/tests/it/proto_conv.rs

@@ -52,6 +52,7 @@ fn new_db_meta_share() -> mt::DatabaseMeta {
         updated_on: Utc.with_ymd_and_hms(2014, 11, 29, 12, 0, 9).unwrap(),
         comment: "foo bar".to_string(),
         drop_on: None,
+        gc_in_progress: false,
     }
 }
 
@@ -64,6 +65,7 @@ fn new_db_meta() -> mt::DatabaseMeta {
         updated_on: Utc.with_ymd_and_hms(2014, 11, 29, 12, 0, 9).unwrap(),
         comment: "foo bar".to_string(),
         drop_on: None,
+        gc_in_progress: false,
     }
 }
 

src/meta/proto-conv/tests/it/v002_database_meta.rs

@@ -48,6 +48,7 @@ fn test_decode_v2_database_meta() -> anyhow::Result<()> {
         updated_on: Utc.with_ymd_and_hms(2014, 11, 29, 12, 0, 9).unwrap(),
         comment: "foo bar".to_string(),
         drop_on: None,
+        gc_in_progress: false,
     };
 
     common::test_pb_from_to(func_name!(), want())?;

src/meta/proto-conv/tests/it/v005_database_meta.rs

@@ -49,6 +49,7 @@ fn test_decode_v5_database_meta() -> anyhow::Result<()> {
         updated_on: Utc.with_ymd_and_hms(2014, 11, 29, 12, 0, 9).unwrap(),
         comment: "foo bar".to_string(),
         drop_on: None,
+        gc_in_progress: false,
     };
 
     common::test_pb_from_to(func_name!(), want())?;

src/meta/proto-conv/tests/it/v055_table_meta.rs

@@ -119,6 +119,7 @@ fn test_decode_v51_database_meta() -> anyhow::Result<()> {
         updated_on: Utc.with_ymd_and_hms(2014, 11, 29, 12, 0, 9).unwrap(),
         comment: "foo bar".to_string(),
         drop_on: None,
+        gc_in_progress: false,
     };
 
     common::test_pb_from_to(func_name!(), want())?;

src/meta/proto-conv/tests/it/v074_table_db_meta.rs

@@ -116,6 +116,7 @@ fn test_decode_v74_database_meta() -> anyhow::Result<()> {
         updated_on: Utc.with_ymd_and_hms(2014, 11, 29, 12, 0, 9).unwrap(),
         comment: "foo bar".to_string(),
         drop_on: None,
+        gc_in_progress: false,
     };
 
     common::test_pb_from_to(func_name!(), want())?;

src/meta/proto-conv/tests/it/v096_database_meta.rs

@@ -39,6 +39,7 @@ fn test_decode_v96_database_meta() -> anyhow::Result<()> {
         updated_on: Utc.with_ymd_and_hms(2014, 11, 29, 12, 0, 9).unwrap(),
         comment: "foo bar".to_string(),
         drop_on: None,
+        gc_in_progress: false,
     };
 
     common::test_pb_from_to(func_name!(), want())?;

src/meta/proto-conv/tests/it/v101_database_meta.rs

@@ -40,6 +40,7 @@ fn v101_database_meta() -> anyhow::Result<()> {
         updated_on: Utc.with_ymd_and_hms(2014, 11, 29, 12, 0, 9).unwrap(),
         comment: "foo bar".to_string(),
         drop_on: None,
+        gc_in_progress: false,
     };
 
     common::test_pb_from_to(func_name!(), want())?;

src/meta/proto-conv/tests/it/v110_database_meta_gc_in_progress.rs

@@ -0,0 +1,60 @@
+// Copyright 2023 Datafuse Labs.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use chrono::TimeZone;
+use chrono::Utc;
+use databend_common_meta_app::schema as mt;
+use fastrace::func_name;
+use maplit::btreemap;
+
+use crate::common;
+
+// These bytes are built when a new version in introduced,
+// and are kept for backward compatibility test.
+//
+// *************************************************************
+// * These messages should never be updated,                   *
+// * only be added when a new version is added,                *
+// * or be removed when an old version is no longer supported. *
+// *************************************************************
+//
+// The message bytes are built from the output of `test_pb_from_to()`
+#[test]
+fn test_decode_v110_database_meta() -> anyhow::Result<()> {
+    let database_meta_v110 = vec![
+        34, 10, 10, 3, 120, 121, 122, 18, 3, 102, 111, 111, 42, 2, 52, 52, 50, 10, 10, 3, 97, 98,
+        99, 18, 3, 100, 101, 102, 162, 1, 23, 50, 48, 49, 52, 45, 49, 49, 45, 50, 56, 32, 49, 50,
+        58, 48, 48, 58, 48, 57, 32, 85, 84, 67, 170, 1, 23, 50, 48, 49, 52, 45, 49, 49, 45, 50, 57,
+        32, 49, 50, 58, 48, 48, 58, 48, 57, 32, 85, 84, 67, 178, 1, 7, 102, 111, 111, 32, 98, 97,
+        114, 232, 1, 1, 160, 6, 110, 168, 6, 24,
+    ];
+
+    let want = || mt::DatabaseMeta {
+        engine: "44".to_string(),
+        engine_options: btreemap! {s("abc") => s("def")},
+        options: btreemap! {s("xyz") => s("foo")},
+        created_on: Utc.with_ymd_and_hms(2014, 11, 28, 12, 0, 9).unwrap(),
+        updated_on: Utc.with_ymd_and_hms(2014, 11, 29, 12, 0, 9).unwrap(),
+        comment: "foo bar".to_string(),
+        drop_on: None,
+        gc_in_progress: true,
+    };
+
+    common::test_pb_from_to(func_name!(), want())?;
+    common::test_load_old(func_name!(), database_meta_v110.as_slice(), 110, want())
+}
+
+fn s(ss: impl ToString) -> String {
+    ss.to_string()
+}

src/meta/protos/proto/database.proto

@@ -70,6 +70,11 @@ message DatabaseMeta {
   // The time table dropped.
   optional string drop_on = 23;
 
+  // Indicates whether garbage collection is currently in progress for this dropped database.
+  //
+  // If it is in progress, the database should not be un-dropped, because the data may be incomplete.
+  bool gc_in_progress = 29;
+
   repeated uint64 shared_by = 24;
 
   optional TIdent from_share = 25;