Skip to content

Commit 8f6713d

Browse files
TCeasonTCeason
authored
refactor: Optimize When Querying the tables_history Table with the Condition table_id = n (#17166)
* optimize: Optimize When Querying the tables_history Table with the Condition table_id = n * use btreeset replace vec * ignore check contain and add some comment
1 parent acd8e61 commit 8f6713d

File tree

14 files changed

+61
-37
lines changed

14 files changed

+61
-37
lines changed

‎src/meta/api/src/schema_api.rs

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -242,6 +242,7 @@ pub trait SchemaApi: Send + Sync {
242242
async fn mget_table_names_by_ids(
243243
&self,
244244
table_ids: &[MetaId],
245+
get_dropped_table: bool,
245246
) -> Result<Vec<Option<String>>, KVAppError>;
246247

247248
async fn mget_database_names_by_ids(

‎src/meta/api/src/schema_api_impl.rs

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1639,6 +1639,7 @@ impl<KV: kvapi::KVApi<Error = MetaError> + ?Sized> SchemaApi for KV {
16391639
async fn mget_table_names_by_ids(
16401640
&self,
16411641
table_ids: &[MetaId],
1642+
get_dropped_table: bool,
16421643
) -> Result<Vec<Option<String>>, KVAppError> {
16431644
debug!(req :? =(&table_ids); "SchemaApi: {}", func_name!());
16441645

@@ -1654,7 +1655,7 @@ impl<KV: kvapi::KVApi<Error = MetaError> + ?Sized> SchemaApi for KV {
16541655
let seq_metas = self.get_pb_values_vec(id_idents).await?;
16551656
for (i, seq_meta_opt) in seq_metas.iter().enumerate() {
16561657
if let Some(seq_meta) = seq_meta_opt {
1657-
if seq_meta.data.drop_on.is_some() {
1658+
if seq_meta.data.drop_on.is_some() && !get_dropped_table {
16581659
table_names[i] = None;
16591660
}
16601661
} else {

‎src/query/catalog/src/catalog/interface.rs

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -216,6 +216,7 @@ pub trait Catalog: DynClone + Send + Sync + Debug {
216216
&self,
217217
tenant: &Tenant,
218218
table_ids: &[MetaId],
219+
get_dropped_table: bool,
219220
) -> Result<Vec<Option<String>>>;
220221

221222
// Get the db name by meta id.

‎src/query/service/src/catalogs/default/database_catalog.rs

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -288,14 +288,15 @@ impl Catalog for DatabaseCatalog {
288288
&self,
289289
tenant: &Tenant,
290290
table_ids: &[MetaId],
291+
get_dropped_table: bool,
291292
) -> Result<Vec<Option<String>>> {
292293
let sys_table_names = self
293294
.immutable_catalog
294-
.mget_table_names_by_ids(tenant, table_ids)
295+
.mget_table_names_by_ids(tenant, table_ids, get_dropped_table)
295296
.await?;
296297
let mut_table_names = self
297298
.mutable_catalog
298-
.mget_table_names_by_ids(tenant, table_ids)
299+
.mget_table_names_by_ids(tenant, table_ids, get_dropped_table)
299300
.await?;
300301

301302
let mut table_names = Vec::with_capacity(table_ids.len());

‎src/query/service/src/catalogs/default/immutable_catalog.rs

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -219,6 +219,7 @@ impl Catalog for ImmutableCatalog {
219219
&self,
220220
_tenant: &Tenant,
221221
table_ids: &[MetaId],
222+
_get_dropped_table: bool,
222223
) -> Result<Vec<Option<String>>> {
223224
let mut table_name = Vec::with_capacity(table_ids.len());
224225
for id in table_ids {

‎src/query/service/src/catalogs/default/mutable_catalog.rs

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -447,8 +447,13 @@ impl Catalog for MutableCatalog {
447447
&self,
448448
_tenant: &Tenant,
449449
table_ids: &[MetaId],
450+
get_dropped_table: bool,
450451
) -> Result<Vec<Option<String>>> {
451-
let res = self.ctx.meta.mget_table_names_by_ids(table_ids).await?;
452+
let res = self
453+
.ctx
454+
.meta
455+
.mget_table_names_by_ids(table_ids, get_dropped_table)
456+
.await?;
452457
Ok(res)
453458
}
454459

‎src/query/service/src/catalogs/default/session_catalog.rs

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -289,8 +289,11 @@ impl Catalog for SessionCatalog {
289289
&self,
290290
tenant: &Tenant,
291291
table_ids: &[MetaId],
292+
get_dropped_table: bool,
292293
) -> databend_common_exception::Result<Vec<Option<String>>> {
293-
self.inner.mget_table_names_by_ids(tenant, table_ids).await
294+
self.inner
295+
.mget_table_names_by_ids(tenant, table_ids, get_dropped_table)
296+
.await
294297
}
295298

296299
async fn get_table_name_by_id(&self, table_id: MetaId) -> Result<Option<String>> {

‎src/query/service/src/table_functions/show_grants/show_grants_table.rs

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -555,7 +555,9 @@ async fn show_account_grants(
555555
.collect::<HashSet<u64>>();
556556
let mut table_ids = table_id_set.into_iter().collect::<Vec<u64>>();
557557
table_ids.sort();
558-
let table_names = catalog.mget_table_names_by_ids(&tenant, &table_ids).await?;
558+
let table_names = catalog
559+
.mget_table_names_by_ids(&tenant, &table_ids, false)
560+
.await?;
559561
let table_map = table_ids
560562
.into_iter()
561563
.zip(table_names.into_iter())

‎src/query/service/tests/it/sql/exec/get_table_bind_test.rs

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -203,8 +203,11 @@ impl Catalog for FakedCatalog {
203203
&self,
204204
tenant: &Tenant,
205205
table_ids: &[MetaId],
206+
get_dropped_table: bool,
206207
) -> Result<Vec<Option<String>>> {
207-
self.cat.mget_table_names_by_ids(tenant, table_ids).await
208+
self.cat
209+
.mget_table_names_by_ids(tenant, table_ids, get_dropped_table)
210+
.await
208211
}
209212

210213
async fn get_db_name_by_id(&self, db_id: MetaId) -> Result<String> {

‎src/query/service/tests/it/storages/fuse/operations/commit.rs

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -940,8 +940,11 @@ impl Catalog for FakedCatalog {
940940
&self,
941941
tenant: &Tenant,
942942
table_id: &[MetaId],
943+
get_dropped_table: bool,
943944
) -> Result<Vec<Option<String>>> {
944-
self.cat.mget_table_names_by_ids(tenant, table_id).await
945+
self.cat
946+
.mget_table_names_by_ids(tenant, table_id, get_dropped_table)
947+
.await
945948
}
946949

947950
async fn get_table_name_by_id(&self, table_id: MetaId) -> Result<Option<String>> {

‎src/query/storages/hive/hive/src/hive_catalog.rs

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -372,6 +372,7 @@ impl Catalog for HiveCatalog {
372372
&self,
373373
_tenant: &Tenant,
374374
_table_ids: &[MetaId],
375+
_get_dropped_table: bool,
375376
) -> Result<Vec<Option<String>>> {
376377
Err(ErrorCode::Unimplemented(
377378
"Cannot get tables name by ids in HIVE catalog",

‎src/query/storages/iceberg/src/catalog.rs

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -309,6 +309,7 @@ impl Catalog for IcebergCatalog {
309309
&self,
310310
_tenant: &Tenant,
311311
_table_ids: &[MetaId],
312+
_get_dropped_table: bool,
312313
) -> Result<Vec<Option<String>>> {
313314
Err(ErrorCode::Unimplemented(
314315
"Cannot get tables name by ids in HIVE catalog",

‎src/query/storages/system/src/streams_table.rs

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -297,7 +297,9 @@ impl<const T: bool> AsyncSystemTable for StreamsTable<T> {
297297

298298
let mut source_tb_ids = source_tb_id_set.into_iter().collect::<Vec<u64>>();
299299
source_tb_ids.sort();
300-
let source_tb_names = ctl.mget_table_names_by_ids(&tenant, &source_tb_ids).await?;
300+
let source_tb_names = ctl
301+
.mget_table_names_by_ids(&tenant, &source_tb_ids, false)
302+
.await?;
301303
let source_tb_map = source_tb_ids
302304
.into_iter()
303305
.zip(source_tb_names.into_iter())

‎src/query/storages/system/src/tables_table.rs

Lines changed: 27 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@
1212
// See the License for the specific language governing permissions and
1313
// limitations under the License.
1414

15+
use std::collections::BTreeSet;
1516
use std::collections::HashMap;
1617
use std::collections::HashSet;
1718
use std::sync::Arc;
@@ -252,7 +253,7 @@ where TablesTable<WITH_HISTORY, WITHOUT_VIEW>: HistoryAware
252253
let user_api = UserApiProvider::instance();
253254

254255
let mut dbs = Vec::new();
255-
let mut tables_names: Vec<String> = Vec::new();
256+
let mut tables_names: BTreeSet<String> = BTreeSet::new();
256257
let mut invalid_tables_ids = false;
257258
let mut tables_ids: Vec<u64> = Vec::new();
258259
let mut db_name: Vec<String> = Vec::new();
@@ -317,9 +318,7 @@ where TablesTable<WITH_HISTORY, WITHOUT_VIEW>: HistoryAware
317318
}
318319
} else if col_name == "name" {
319320
if let Scalar::String(t_name) = scalar {
320-
if !tables_names.contains(t_name) {
321-
tables_names.push(t_name.clone());
322-
}
321+
tables_names.insert(t_name.clone());
323322
}
324323
}
325324
Ok(())
@@ -348,17 +347,19 @@ where TablesTable<WITH_HISTORY, WITHOUT_VIEW>: HistoryAware
348347
}
349348
}
350349
}
351-
if let Err(err) = ctl.mget_table_names_by_ids(&tenant, &tables_ids).await {
352-
warn!("Failed to get tables: {}, {}", ctl.name(), err);
353-
} else {
354-
let new_tables_names = ctl
355-
.mget_table_names_by_ids(&tenant, &tables_ids)
356-
.await?
357-
.into_iter()
358-
.flatten()
359-
.filter(|table| !tables_names.contains(table))
360-
.collect::<Vec<_>>();
361-
tables_names.extend(new_tables_names);
350+
match ctl
351+
.mget_table_names_by_ids(&tenant, &tables_ids, false)
352+
.await
353+
{
354+
Ok(new_tables) => {
355+
let new_table_names: BTreeSet<_> =
356+
new_tables.into_iter().flatten().collect();
357+
tables_names.extend(new_table_names);
358+
}
359+
Err(err) => {
360+
// swallow the errors related with mget tables
361+
warn!("Failed to get tables: {}, {}", ctl.name(), err);
362+
}
362363
}
363364

364365
for table_name in &tables_names {
@@ -430,21 +431,19 @@ where TablesTable<WITH_HISTORY, WITHOUT_VIEW>: HistoryAware
430431
}
431432
}
432433

433-
if !WITH_HISTORY {
434-
match ctl.mget_table_names_by_ids(&tenant, &tables_ids).await {
435-
Ok(tables) => {
436-
for table in tables.into_iter().flatten() {
437-
if !tables_names.contains(&table) {
438-
tables_names.push(table.clone());
439-
}
440-
}
441-
}
442-
Err(err) => {
443-
let msg =
444-
format!("Failed to get tables: {}, {}", ctl.name(), err);
445-
warn!("{}", msg);
434+
match ctl
435+
.mget_table_names_by_ids(&tenant, &tables_ids, WITH_HISTORY)
436+
.await
437+
{
438+
Ok(tables) => {
439+
for table in tables.into_iter().flatten() {
440+
tables_names.insert(table.clone());
446441
}
447442
}
443+
Err(err) => {
444+
let msg = format!("Failed to get tables: {}, {}", ctl.name(), err);
445+
warn!("{}", msg);
446+
}
448447
}
449448
}
450449
}

0 commit comments

Comments
 (0)