Remove source_url from data_sources_documents (#10085)

* read the source_url from data_sources_nodes instead of data_sources_documents * prevent writes to data_sources_documents.source_url * add a migration script * remove the column from the init_db
dust-tt · Jan 27, 2025 · 217e0d4 · 217e0d4
1 parent bae93cd
commit 217e0d4
Show file tree

Hide file tree

Showing 3 changed files with 5 additions and 6 deletions.
diff --git a/core/src/stores/migrations/20250118_drop_document_source_url_column.sql b/core/src/stores/migrations/20250118_drop_document_source_url_column.sql
@@ -0,0 +1 @@
+ALTER TABLE data_sources_documents DROP COLUMN IF EXISTS source_url;
diff --git a/core/src/stores/postgres.rs b/core/src/stores/postgres.rs
@@ -1336,7 +1336,7 @@ impl Store for PostgresStore {
             None => {
                 c.query(
                     "SELECT dsd.id, dsd.created, dsd.timestamp, dsd.tags_array, dsn.parents, \
-                       dsd.source_url, dsd.hash, dsd.text_size, dsd.chunk_count, dsn.title, \
+                       dsn.source_url, dsd.hash, dsd.text_size, dsd.chunk_count, dsn.title, \
                        dsn.mime_type, dsn.provider_visibility \
                        FROM data_sources_documents dsd \
                        INNER JOIN data_sources_nodes dsn ON dsn.document=dsd.id \
@@ -1349,7 +1349,7 @@ impl Store for PostgresStore {
             Some(version_hash) => {
                 c.query(
                     "SELECT dsd.id, dsd.created, dsd.timestamp, dsd.tags_array, dsn.parents, \
-                       dsd.source_url, dsd.hash, dsd.text_size, dsd.chunk_count, dsn.title, \
+                       dsn.source_url, dsd.hash, dsd.text_size, dsd.chunk_count, dsn.title, \
                        dsn.mime_type, dsn.provider_visibility \
                        FROM data_sources_documents dsd \
                        INNER JOIN data_sources_nodes dsn ON dsn.document=dsd.id \
@@ -1867,7 +1867,7 @@ impl Store for PostgresStore {
             .prepare(
                 "INSERT INTO data_sources_documents \
                    (id, data_source, created, document_id, timestamp, tags_array, \
-                    source_url, hash, text_size, chunk_count, status) \
+                    hash, text_size, chunk_count, status) \
                    VALUES (DEFAULT, $1, $2, $3, $4, $5, $6, $7, $8, $9, $10) \
                    RETURNING id, created",
             )
@@ -1882,7 +1882,6 @@ impl Store for PostgresStore {
                     &create_params.document_id,
                     &(create_params.timestamp as i64),
                     &create_params.tags,
-                    &create_params.source_url,
                     &create_params.hash,
                     &(create_params.text_size as i64),
                     &(create_params.chunk_count as i64),
@@ -2009,7 +2008,7 @@ impl Store for PostgresStore {
 
         let sql = format!(
             "SELECT dsd.id, dsd.created, dsd.document_id, dsd.timestamp, dsd.tags_array, \
-               dsn.parents, dsd.source_url, dsd.hash, dsd.text_size, dsd.chunk_count, \
+               dsn.parents, dsn.source_url, dsd.hash, dsd.text_size, dsd.chunk_count, \
                dsn.title, dsn.mime_type, dsn.provider_visibility \
                FROM data_sources_documents dsd \
                INNER JOIN data_sources_nodes dsn ON dsn.document=dsd.id \

diff --git a/core/src/stores/store.rs b/core/src/stores/store.rs
@@ -547,7 +547,6 @@ pub const POSTGRES_TABLES: [&'static str; 16] = [
        document_id              TEXT NOT NULL,
        timestamp                BIGINT NOT NULL,
        tags_array               TEXT[] NOT NULL,
-       source_url               TEXT,
        hash                     TEXT NOT NULL,
        text_size                BIGINT NOT NULL,
        chunk_count              BIGINT NOT NULL,
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1 @@
		ALTER TABLE data_sources_documents DROP COLUMN IF EXISTS source_url;