From 8b5279924f2226ed278993d2265bcf417703aa11 Mon Sep 17 00:00:00 2001 From: shiyasmohd Date: Mon, 3 Feb 2025 07:06:56 +0530 Subject: [PATCH 1/3] store: pass list of indexes when pruning a subgraph --- store/postgres/src/relational/index.rs | 16 ++++++++++++---- store/postgres/src/relational/prune.rs | 4 ++-- 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/store/postgres/src/relational/index.rs b/store/postgres/src/relational/index.rs index 6013a5d9e68..4cc354b8e89 100644 --- a/store/postgres/src/relational/index.rs +++ b/store/postgres/src/relational/index.rs @@ -734,6 +734,16 @@ pub struct IndexList { pub(crate) indexes: HashMap>, } +pub fn load_indexes_from_table( + conn: &mut PgConnection, + table: &Arc, + schema_name: &str, +) -> Result, StoreError> { + let table_name = table.name.as_str(); + let indexes = catalog::indexes_for_table(conn, schema_name, table_name)?; + Ok(indexes.into_iter().map(CreateIndex::parse).collect()) +} + impl IndexList { pub fn load( conn: &mut PgConnection, @@ -746,10 +756,8 @@ impl IndexList { let schema_name = site.namespace.clone(); let layout = store.layout(conn, site)?; for (_, table) in &layout.tables { - let table_name = table.name.as_str(); - let indexes = catalog::indexes_for_table(conn, schema_name.as_str(), table_name)?; - let collect: Vec = indexes.into_iter().map(CreateIndex::parse).collect(); - list.indexes.insert(table_name.to_string(), collect); + let indexes = load_indexes_from_table(conn, table, schema_name.as_str())?; + list.indexes.insert(table.name.to_string(), indexes); } Ok(list) } diff --git a/store/postgres/src/relational/prune.rs b/store/postgres/src/relational/prune.rs index 62632549397..f3aba1e4301 100644 --- a/store/postgres/src/relational/prune.rs +++ b/store/postgres/src/relational/prune.rs @@ -1,4 +1,4 @@ -use std::{fmt::Write, sync::Arc}; +use std::{collections::HashMap, fmt::Write, sync::Arc, time::Instant}; use diesel::{ connection::SimpleConnection, @@ -58,7 +58,7 @@ impl TablePair { } else { // In case of pruning we don't do delayed creation of indexes, // as the asumption is that there is not that much data inserted. - dst.as_ddl(schema, catalog, None, &mut query)?; + dst.as_ddl(schema, catalog, Some(&list), &mut query)?; } conn.batch_execute(&query)?; From c4ad7280dd4aafc927d347fa7f8d4e56d066a6cd Mon Sep 17 00:00:00 2001 From: shiyasmohd Date: Tue, 11 Feb 2025 09:54:53 +0530 Subject: [PATCH 2/3] store: fix namespace reference when loading indexes from table --- store/postgres/src/relational/prune.rs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/store/postgres/src/relational/prune.rs b/store/postgres/src/relational/prune.rs index f3aba1e4301..10d0c7c93f9 100644 --- a/store/postgres/src/relational/prune.rs +++ b/store/postgres/src/relational/prune.rs @@ -56,6 +56,12 @@ impl TablePair { if catalog::table_exists(conn, dst_nsp.as_str(), &dst.name)? { writeln!(query, "truncate table {};", dst.qualified_name)?; } else { + let mut list = IndexList { + indexes: HashMap::new(), + }; + let indexes = load_indexes_from_table(conn, &src, src_nsp.as_str())?; + list.indexes.insert(src.name.to_string(), indexes); + // In case of pruning we don't do delayed creation of indexes, // as the asumption is that there is not that much data inserted. dst.as_ddl(schema, catalog, Some(&list), &mut query)?; From 74d02c694d304cf26fb3037004a0ccec3abd0022 Mon Sep 17 00:00:00 2001 From: shiyasmohd Date: Thu, 20 Feb 2025 20:08:28 +0530 Subject: [PATCH 3/3] store: update namespace handling in table creation and index loading --- store/postgres/src/relational.rs | 2 +- store/postgres/src/relational/ddl.rs | 8 +------- store/postgres/src/relational/index.rs | 2 +- store/postgres/src/relational/prune.rs | 12 +++++++++--- 4 files changed, 12 insertions(+), 12 deletions(-) diff --git a/store/postgres/src/relational.rs b/store/postgres/src/relational.rs index de7e6895083..efb5776adf1 100644 --- a/store/postgres/src/relational.rs +++ b/store/postgres/src/relational.rs @@ -1715,7 +1715,7 @@ impl Table { pub fn new_like(&self, namespace: &Namespace, name: &SqlName) -> Arc
{ let other = Table { object: self.object.clone(), - nsp: self.nsp.clone(), + nsp: namespace.clone(), name: name.clone(), qualified_name: SqlName::qualified_name(namespace, name), columns: self.columns.clone(), diff --git a/store/postgres/src/relational/ddl.rs b/store/postgres/src/relational/ddl.rs index 40a02d6051e..55e116272d1 100644 --- a/store/postgres/src/relational/ddl.rs +++ b/store/postgres/src/relational/ddl.rs @@ -403,13 +403,7 @@ impl Table { if index_def.is_some() && ENV_VARS.postpone_attribute_index_creation { let arr = index_def .unwrap() - .indexes_for_table( - &catalog.site.namespace, - &self.name.to_string(), - &self, - false, - false, - ) + .indexes_for_table(&self.nsp, &self.name.to_string(), &self, false, false) .map_err(|_| fmt::Error)?; for (_, sql) in arr { writeln!(out, "{};", sql).expect("properly formated index statements") diff --git a/store/postgres/src/relational/index.rs b/store/postgres/src/relational/index.rs index 4cc354b8e89..4f72e773ee6 100644 --- a/store/postgres/src/relational/index.rs +++ b/store/postgres/src/relational/index.rs @@ -440,7 +440,7 @@ impl CreateIndex { } } - fn with_nsp(&self, nsp2: String) -> Result { + pub fn with_nsp(&self, nsp2: String) -> Result { let s = self.clone(); match s { CreateIndex::Unknown { defn: _ } => Err(anyhow!("Failed to parse the index")), diff --git a/store/postgres/src/relational/prune.rs b/store/postgres/src/relational/prune.rs index 10d0c7c93f9..5c3035ce172 100644 --- a/store/postgres/src/relational/prune.rs +++ b/store/postgres/src/relational/prune.rs @@ -1,4 +1,4 @@ -use std::{collections::HashMap, fmt::Write, sync::Arc, time::Instant}; +use std::{collections::HashMap, fmt::Write, sync::Arc}; use diesel::{ connection::SimpleConnection, @@ -23,7 +23,10 @@ use crate::{ vid_batcher::{VidBatcher, VidRange}, }; -use super::{Catalog, Layout, Namespace}; +use super::{ + index::{load_indexes_from_table, CreateIndex, IndexList}, + Catalog, Layout, Namespace, +}; /// Utility to copy relevant data out of a source table and into a new /// destination table and replace the source table with the destination @@ -59,7 +62,10 @@ impl TablePair { let mut list = IndexList { indexes: HashMap::new(), }; - let indexes = load_indexes_from_table(conn, &src, src_nsp.as_str())?; + let indexes = load_indexes_from_table(conn, &src, src_nsp.as_str())? + .into_iter() + .map(|index| index.with_nsp(dst_nsp.to_string())) + .collect::, _>>()?; list.indexes.insert(src.name.to_string(), indexes); // In case of pruning we don't do delayed creation of indexes,