Skip to content

Commit f72f0fd

Browse files
alexdachinalexdachin
committed
Allow external blobs
1 parent 76b066b commit f72f0fd

File tree

24 files changed

+291
-65
lines changed

24 files changed

+291
-65
lines changed

‎apps/server/spec/db/document.db‎

8 KB
Binary file not shown.

‎apps/server/src/assets/db/schema.sql‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -107,6 +107,8 @@ CREATE TABLE IF NOT EXISTS "recent_notes"
107107
CREATE TABLE IF NOT EXISTS "blobs" (
108108
`blobId` TEXT NOT NULL,
109109
`content` TEXT NULL DEFAULT NULL,
110+
`contentLocation` TEXT NOT NULL DEFAULT 'internal',
111+
`contentLength` INTEGER NOT NULL DEFAULT 0,
110112
`dateModified` TEXT NOT NULL,
111113
`utcDateModified` TEXT NOT NULL,
112114
PRIMARY KEY(`blobId`)

‎apps/server/src/becca/becca-interface.ts‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -171,7 +171,7 @@ export default class Becca {
171171
opts.includeContentLength = !!opts.includeContentLength;
172172

173173
const query = opts.includeContentLength
174-
? /*sql*/`SELECT attachments.*, LENGTH(blobs.content) AS contentLength
174+
? /*sql*/`SELECT attachments.*, blobs.contentLength AS contentLength
175175
FROM attachments
176176
JOIN blobs USING (blobId)
177177
WHERE attachmentId = ? AND isDeleted = 0`
@@ -197,7 +197,7 @@ export default class Becca {
197197
return null;
198198
}
199199

200-
const row = sql.getRow<BlobRow | null>("SELECT *, LENGTH(content) AS contentLength FROM blobs WHERE blobId = ?", [entity.blobId]);
200+
const row = sql.getRow<BlobRow | null>("SELECT * FROM blobs WHERE blobId = ?", [entity.blobId]);
201201
return row ? new BBlob(row) : null;
202202
}
203203

‎apps/server/src/becca/entities/abstract_becca_entity.ts‎

Lines changed: 62 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -9,8 +9,11 @@ import cls from "../../services/cls.js";
99
import log from "../../services/log.js";
1010
import protectedSessionService from "../../services/protected_session.js";
1111
import blobService from "../../services/blob.js";
12+
import blobStorageService from "../../services/blob-storage.js";
13+
import type { Blob } from "../../services/blob-interface.js";
1214
import type { default as Becca, ConstructorData } from "../becca-interface.js";
1315
import becca from "../becca.js";
16+
import type { BlobContentLocation, BlobRow } from "@triliumnext/commons";
1417

1518
interface ContentOpts {
1619
forceSave?: boolean;
@@ -195,6 +198,21 @@ abstract class AbstractBeccaEntity<T extends AbstractBeccaEntity<T>> {
195198
return;
196199
}
197200

201+
try {
202+
const row = sql.getRow<{ contentLocation: string }>("SELECT contentLocation FROM blobs WHERE blobId = ?", [oldBlobId]);
203+
if (row?.contentLocation.startsWith('file://')) {
204+
const filePath = row.contentLocation.replace('file://', '');
205+
blobStorageService.deleteExternal(filePath);
206+
}
207+
} catch (error) {
208+
// contentLocation column might not be present when applying older migrations
209+
if (error instanceof Error && error.name === 'SqliteError' && error.message.includes("no such column: contentLocation")) {
210+
// ignore
211+
} else {
212+
log.error(`Failed to delete external content file for ${oldBlobId}: ${error}`);
213+
}
214+
}
215+
198216
sql.execute("DELETE FROM blobs WHERE blobId = ?", [oldBlobId]);
199217
// blobs are not marked as erased in entity_changes, they are just purged completely
200218
// this is because technically every keystroke can create a new blob, and there would be just too many
@@ -225,14 +243,41 @@ abstract class AbstractBeccaEntity<T extends AbstractBeccaEntity<T>> {
225243
return newBlobId;
226244
}
227245

228-
const pojo = {
246+
// Check if we should store this blob externally
247+
const shouldStoreExternally = blobStorageService.shouldStoreExternally(content);
248+
let contentLocation: BlobContentLocation = 'internal';
249+
if (shouldStoreExternally) {
250+
try {
251+
const filePath = blobStorageService.saveExternal(newBlobId, content);
252+
contentLocation = `file://${filePath}` as BlobContentLocation;
253+
} catch (error) {
254+
log.error(`Failed to store blob ${newBlobId} externally, falling back to internal storage: ${error}`);
255+
contentLocation = 'internal';
256+
}
257+
}
258+
259+
const contentLength = blobService.getContentLength(content);
260+
261+
const pojo: BlobRow = {
229262
blobId: newBlobId,
230-
content: content,
263+
content: contentLocation === 'internal' ? content : null,
264+
contentLocation,
265+
contentLength,
231266
dateModified: dateUtils.localNowDateTime(),
232267
utcDateModified: dateUtils.utcNowDateTime()
233268
};
234269

235-
sql.upsert("blobs", "blobId", pojo);
270+
// external content columns might not be present when applying older migrations
271+
const hasExternalContentColumns = sql.getValue("SELECT 1 FROM pragma_table_info('blobs') WHERE name = 'contentLocation'");
272+
const pojoToSave = hasExternalContentColumns
273+
? pojo
274+
: {
275+
blobId: pojo.blobId,
276+
content,
277+
dateModified: pojo.dateModified,
278+
utcDateModified: pojo.utcDateModified
279+
};
280+
sql.upsert("blobs", "blobId", pojoToSave);
236281

237282
// we can't reuse blobId as an entity_changes hash, because this one has to be calculatable without having
238283
// access to the decrypted content
@@ -259,14 +304,26 @@ abstract class AbstractBeccaEntity<T extends AbstractBeccaEntity<T>> {
259304
}
260305

261306
protected _getContent(): string | Buffer {
262-
const row = sql.getRow<{ content: string | Buffer }>(/*sql*/`SELECT content FROM blobs WHERE blobId = ?`, [this.blobId]);
307+
let row: { content: string | Buffer, contentLocation: string };
308+
try {
309+
row = sql.getRow<{ content: string | Buffer, contentLocation: string }>(/*sql*/`SELECT content, contentLocation FROM blobs WHERE blobId = ?`, [this.blobId]);
310+
} catch (error) {
311+
// contentLocation column might not be present when applying older migrations
312+
if (error instanceof Error && error.name === 'SqliteError' && error.message.includes("no such column: contentLocation")) {
313+
row = sql.getRow<{ content: string | Buffer, contentLocation: string }>(/*sql*/`SELECT content, 'internal' as contentLocation FROM blobs WHERE blobId = ?`, [this.blobId]);
314+
} else {
315+
throw error;
316+
}
317+
}
263318

264319
if (!row) {
265320
const constructorData = this.constructor as unknown as ConstructorData<T>;
266321
throw new Error(`Cannot find content for ${constructorData.primaryKeyName} '${(this as any)[constructorData.primaryKeyName]}', blobId '${this.blobId}'`);
267322
}
268323

269-
return blobService.processContent(row.content, this.isProtected || false, this.hasStringContent());
324+
const content = blobStorageService.getContent(row);
325+
326+
return blobService.processContent(content, this.isProtected || false, this.hasStringContent());
270327
}
271328

272329
/**

‎apps/server/src/becca/entities/bblob.ts‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
import AbstractBeccaEntity from "./abstract_becca_entity.js";
2-
import type { BlobRow } from "@triliumnext/commons";
2+
import type { BlobRow, BlobContentLocation } from "@triliumnext/commons";
33

44
// TODO: Why this does not extend the abstract becca?
55
class BBlob extends AbstractBeccaEntity<BBlob> {
@@ -10,11 +10,12 @@ class BBlob extends AbstractBeccaEntity<BBlob> {
1010
return "blobId";
1111
}
1212
static get hashedProperties() {
13-
return ["blobId", "content"];
13+
return ["blobId", "content", "contentLocation"];
1414
}
1515

16-
content!: string | Buffer;
16+
content!: string | Buffer | null;
1717
contentLength!: number;
18+
contentLocation!: BlobContentLocation;
1819

1920
constructor(row: BlobRow) {
2021
super();
@@ -25,6 +26,7 @@ class BBlob extends AbstractBeccaEntity<BBlob> {
2526
this.blobId = row.blobId;
2627
this.content = row.content;
2728
this.contentLength = row.contentLength;
29+
this.contentLocation = row.contentLocation;
2830
this.dateModified = row.dateModified;
2931
this.utcDateModified = row.utcDateModified;
3032
}
@@ -34,6 +36,7 @@ class BBlob extends AbstractBeccaEntity<BBlob> {
3436
blobId: this.blobId,
3537
content: this.content || null,
3638
contentLength: this.contentLength,
39+
contentLocation: this.contentLocation,
3740
dateModified: this.dateModified,
3841
utcDateModified: this.utcDateModified
3942
};

‎apps/server/src/becca/entities/bnote.ts‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1108,7 +1108,7 @@ class BNote extends AbstractBeccaEntity<BNote> {
11081108
// given that we're always fetching attachments only for a specific note, we might just do it always
11091109

11101110
const query = opts.includeContentLength
1111-
? /*sql*/`SELECT attachments.*, LENGTH(blobs.content) AS contentLength
1111+
? /*sql*/`SELECT attachments.*, blobs.contentLength AS contentLength
11121112
FROM attachments
11131113
JOIN blobs USING (blobId)
11141114
WHERE ownerId = ? AND isDeleted = 0
@@ -1122,7 +1122,7 @@ class BNote extends AbstractBeccaEntity<BNote> {
11221122
opts.includeContentLength = !!opts.includeContentLength;
11231123

11241124
const query = opts.includeContentLength
1125-
? /*sql*/`SELECT attachments.*, LENGTH(blobs.content) AS contentLength
1125+
? /*sql*/`SELECT attachments.*, blobs.contentLength AS contentLength
11261126
FROM attachments
11271127
JOIN blobs USING (blobId)
11281128
WHERE ownerId = ? AND attachmentId = ? AND isDeleted = 0`

‎apps/server/src/becca/entities/brevision.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -141,7 +141,7 @@ class BRevision extends AbstractBeccaEntity<BRevision> {
141141
opts.includeContentLength = !!opts.includeContentLength;
142142

143143
const query = opts.includeContentLength
144-
? /*sql*/`SELECT attachments.*, LENGTH(blobs.content) AS contentLength
144+
? /*sql*/`SELECT attachments.*, blobs.contentLength AS contentLength
145145
FROM attachments
146146
JOIN blobs USING (blobId)
147147
WHERE ownerId = ? AND attachmentId = ? AND isDeleted = 0`

‎apps/server/src/migrations/migrations.ts‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,19 @@
66

77
// Migrations should be kept in descending order, so the latest migration is first.
88
const MIGRATIONS: (SqlMigration | JsMigration)[] = [
9+
// Add external blob storage support
10+
{
11+
version: 234,
12+
sql: /*sql*/`
13+
-- Add contentLocation column
14+
ALTER TABLE blobs ADD contentLocation TEXT DEFAULT 'internal';
15+
UPDATE blobs SET contentLocation = 'internal' WHERE contentLocation IS NULL;
16+
17+
-- Add contentLength column
18+
ALTER TABLE blobs ADD contentLength INTEGER DEFAULT 0;
19+
UPDATE blobs SET contentLength = CASE WHEN content IS NULL THEN 0 ELSE LENGTH(content) END WHERE contentLength IS NULL;
20+
`,
21+
},
922
// Migrate geo map to collection
1023
{
1124
version: 233,

‎apps/server/src/routes/api/revisions.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,7 @@ function getRevisions(req: Request) {
3636
return becca.getRevisionsFromQuery(
3737
`
3838
SELECT revisions.*,
39-
LENGTH(blobs.content) AS contentLength
39+
blobs.contentLength AS contentLength
4040
FROM revisions
4141
JOIN blobs ON revisions.blobId = blobs.blobId
4242
WHERE revisions.noteId = ?

‎apps/server/src/routes/api/stats.ts‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@ function getNoteSize(req: Request) {
77

88
const blobSizes = sql.getMap<string, number>(
99
`
10-
SELECT blobs.blobId, LENGTH(content)
10+
SELECT blobs.blobId, blobs.contentLength
1111
FROM blobs
1212
LEFT JOIN notes ON notes.blobId = blobs.blobId AND notes.noteId = ? AND notes.isDeleted = 0
1313
LEFT JOIN attachments ON attachments.blobId = blobs.blobId AND attachments.ownerId = ? AND attachments.isDeleted = 0
@@ -33,7 +33,7 @@ function getSubtreeSize(req: Request) {
3333
sql.fillParamList(subTreeNoteIds);
3434

3535
const blobSizes = sql.getMap<string, number>(`
36-
SELECT blobs.blobId, LENGTH(content)
36+
SELECT blobs.blobId, blobs.contentLength
3737
FROM param_list
3838
JOIN notes ON notes.noteId = param_list.paramId AND notes.isDeleted = 0
3939
LEFT JOIN attachments ON attachments.ownerId = param_list.paramId AND attachments.isDeleted = 0

0 commit comments

Comments
 (0)