2gb upload limit fix

api/package.json

@@ -39,8 +39,8 @@
     "locutus": "^2",
     "morgan": "~1.10.1",
     "mysql2": "^3.16",
-    "n3": "^2",
-    "nanoid": "^5.1.6",
+    "n3": "^1",
+    "nanoid": "^3.3.11",
     "needle": "^3.1.0",
     "node-tesseract-ocr": "^2",
     "passport": "^0.7.0",

api/src/jobs/Metadata.ts

@@ -1,6 +1,5 @@
 import { Job as QueueJob } from "bullmq";
 import fs = require("fs");
-import tmp = require("tmp");
 import Config from "../models/Config";
 import { FedoraObject } from "../models/FedoraObject";
 import FedoraObjectFactory from "../services/FedoraObjectFactory";
@@ -22,16 +21,18 @@ class MetadataProcessor {
     }
 
     async addMasterMetadataDatastream(): Promise<void> {
+        console.log(`Adding master metadata datastream to ${this.pid}`);
         const fedoraObject: FedoraObject = FedoraObject.build(this.pid, null, this.config);
-        const dataStream: Buffer = await fedoraObject.getDatastreamAsBuffer("MASTER");
-        const contentFile = tmp.fileSync();
-        fs.writeFileSync(contentFile.name, dataStream);
-        await fedoraObject.addMasterMetadataDatastream(contentFile.name);
-        fs.truncateSync(contentFile.name, 0);
-        fs.rmSync(contentFile.name);
+        console.log("FedoraObject.build: Done");
+        // Stream the MASTER datastream directly to a temporary file to avoid
+        // buffering very large files into memory, then run FITS on that file.
+        const contentPath = await fedoraObject.getDatastreamToTempFile("MASTER");
+        await fedoraObject.addMasterMetadataDatastream(contentPath);
+        fs.truncateSync(contentPath, 0);
+        fs.rmSync(contentPath);
         // FITS XML will have been generated in /tmp as a side-effect; clean it up:
-        fs.truncateSync(contentFile.name + ".fits.xml", 0);
-        fs.rmSync(contentFile.name + ".fits.xml");
+        fs.truncateSync(contentPath + ".fits.xml", 0);
+        fs.rmSync(contentPath + ".fits.xml");
     }
 
     async run(): Promise<void> {

api/src/models/FedoraObject.ts

@@ -5,6 +5,7 @@ import Config from "./Config";
 import { DatastreamParameters, Fedora } from "../services/Fedora";
 import FedoraDataCollector from "../services/FedoraDataCollector";
 import { execSync } from "child_process";
+import crypto = require("crypto");
 import { Agent } from "../services/interfaces";
 
 export interface ObjectParameters {
@@ -65,6 +66,7 @@ export class FedoraObject {
             params.logMessage ?? "Adding datastream " + id + " to " + this.pid + " with " + data.length + " bytes",
         );
         await this.fedora.addDatastream(this.pid, id, params, data, expectedStatus);
+        console.log(`Added datastream ${id} to ${this.pid}`);
     }
 
     async deleteDatastream(stream: string): Promise<void> {
@@ -77,7 +79,26 @@ export class FedoraObject {
     }
 
     async addDatastreamFromFile(filename: string, stream: string, mimeType: string): Promise<void> {
-        await this.addDatastreamFromStringOrBuffer(fs.readFileSync(filename), stream, mimeType, [201]);
+        // Compute digest by streaming the file once (avoids loading the whole file into memory)
+        const md5Hash = crypto.createHash("md5");
+        await new Promise<void>((resolve, reject) => {
+            const rs = fs.createReadStream(filename);
+            rs.on("data", (chunk: Buffer) => {
+                md5Hash.update(chunk);
+            });
+            rs.on("end", () => resolve());
+            rs.on("error", (err) => reject(err));
+        });
+        const md5 = md5Hash.digest("hex");
+        const digestHeader = `md5=${md5}`;
+
+        // Create a fresh read stream for the upload
+        const readStream = fs.createReadStream(filename);
+        const params: DatastreamParameters = {
+            mimeType: mimeType,
+            logMessage: "Initial Ingest addDatastream - " + stream,
+        };
+        await this.fedora.addDatastream(this.pid, stream, params, readStream, [201], digestHeader);
     }
 
     async updateDatastreamFromFile(filename: string, stream: string, mimeType: string): Promise<void> {
@@ -105,7 +126,21 @@ export class FedoraObject {
             mimeType: "text/xml",
             logMessage: "Initial Ingest addDatastream - MASTER-MD",
         };
+        console.log("Getting fits MasterMetadata for file:", filename);
         const fitsXml = this.fitsMasterMetadata(filename);
+
+        // Check if MASTER-MD exists and delete it if it does
+        try {
+            const checkResponse = await this.fedora.getDatastream(this.pid, "MASTER-MD");
+            if (checkResponse.statusCode === 200) {
+                console.log("Deleting pre-existing MASTER-MD");
+                await this.deleteDatastream("MASTER-MD");
+            }
+        } catch (e) {
+            console.log("No existing MASTER-MD to delete:", e.message);
+        }
+
+        console.log("Adding MASTER-MD datastream");
         await this.addDatastream("MASTER-MD", params, fitsXml, [201, 204]);
     }
 
@@ -221,11 +256,16 @@ export class FedoraObject {
         return this.fedora.getDatastreamAsBuffer(this.pid, datastream);
     }
 
+    async getDatastreamToTempFile(datastream: string, treatMissingAsEmpty = false): Promise<string> {
+        return this.fedora.downloadDatastreamToTempFile(this.pid, datastream, treatMissingAsEmpty);
+    }
+
     async getDatastreamMetadata(datastream: string): Promise<string> {
         return await this.fedora.getRdf(`${this.pid}/${datastream}/fcr:metadata`);
     }
 
     fitsMasterMetadata(filename: string): string {
+        console.log("Generating FITS metadata for " + filename);
         const targetXml = filename + ".fits.xml";
         if (!fs.existsSync(targetXml)) {
             const fitsCommand = this.config.fitsCommand + " -i " + filename + " -o " + targetXml;