feat: provide minio provider as alternative

Author: aldy505

package-lock.json

@@ -62,7 +62,8 @@
   "dependencies": {
     "@aws-sdk/client-s3": "^3.620.0",
     "@azure/storage-blob": "^12.24.0",
-    "@google-cloud/storage": "^6.11.0"
+    "@google-cloud/storage": "^6.11.0",
+    "minio": "^8.0.1"
   },
   "directories": {
     "lib": "./src",

package.json

@@ -0,0 +1,135 @@
+import { join } from "node:path";
+import { buffer } from "node:stream/consumers";
+
+import * as Minio from "minio";
+import { Readable, Writable } from "stream";
+
+import { ConnectionString } from "../connectionString";
+import type { IObjectStorage, PutOptions, StatResponse } from "../interface";
+import { UnimplementedError } from "../errors";
+
+export class MinioStorage implements IObjectStorage {
+    private readonly client: Minio.Client;
+    private readonly bucketName: string;
+
+    constructor(config: ConnectionString) {
+        const clientOptions: Minio.ClientOptions = {
+            endPoint: "",
+            accessKey: "",
+            secretKey: ""
+        };
+        if (config.username !== undefined && config.username !== "" && config.password !== undefined && config.password !== "") {
+            clientOptions.accessKey = config.username;
+            clientOptions.secretKey = config.password;
+        }
+
+        if (config.parameters !== undefined) {
+            if ("region" in config.parameters && typeof config.parameters.region === "string") {
+                clientOptions.region = config.parameters.region;
+            } else {
+                // See https://github.com/aws/aws-sdk-js-v3/issues/1845#issuecomment-754832210
+                clientOptions.region = "us-east-1";
+                clientOptions.pathStyle = true;
+            }
+
+            if ("forcePathStyle" in config.parameters) {
+                clientOptions.pathStyle = Boolean(config.parameters.forcePathStyle);
+            }
+
+            if ("useAccelerateEndpoint" in config.parameters) {
+                clientOptions.s3AccelerateEndpoint = config.parameters.useAccelerateEndpoint;
+            }
+
+            if ("endpoint" in config.parameters && typeof config.parameters.endpoint === "string") {
+                clientOptions.endPoint = config.parameters.endpoint;
+            }
+
+            if ("useSSL" in config.parameters) {
+                clientOptions.useSSL = Boolean(config.parameters.useSSL);
+            }
+
+            if ("port" in config.parameters) {
+                clientOptions.port = Number.parseInt(config.parameters.port);
+            }
+        }
+
+        this.client = new Minio.Client(clientOptions);
+        this.bucketName = config.bucketName;
+    }
+
+    async put(path: string, content: string | Buffer, options?: PutOptions | undefined): Promise<void> {
+        await this.client.putObject(this.bucketName, path, content, undefined, options?.metadata);
+    }
+
+    putStream(path: string, options?: PutOptions | undefined): Promise<Writable> {
+        throw new UnimplementedError();
+    }
+
+    async get(path: string, encoding?: string | undefined): Promise<Buffer> {
+        const response = await this.client.getObject(this.bucketName, path);
+        return buffer(response);
+    }
+
+    getStream(path: string): Promise<Readable> {
+        return this.client.getObject(this.bucketName, path);
+    }
+
+    async stat(path: string): Promise<StatResponse> {
+        const response = await this.client.statObject(this.bucketName, path);
+        return {
+            size: response.size,
+            lastModified: response.lastModified,
+            createdTime: new Date(0),
+            etag: response.etag,
+            metadata: response.metaData
+        };
+    }
+
+    list(path?: string | undefined): Promise<Iterable<string>> {
+        return new Promise((resolve, reject) => {
+            const listStream = this.client.listObjectsV2(this.bucketName, path, false);
+            const objects: string[] = [];
+            listStream.on("end", () => {
+                resolve(objects);
+            });
+
+            listStream.on("data", (item) => {
+                if (item.name !== undefined) {
+                    objects.push(item.name);
+                }
+            });
+
+            listStream.on("error", (error) => {
+                reject(error);
+            });
+        });
+    }
+
+    async exists(path: string): Promise<boolean> {
+        try {
+            await this.client.statObject(this.bucketName, path);
+            return true;
+        } catch (error: unknown) {
+            if (error instanceof Minio.S3Error) {
+                if (error.code === "NoSuchKey") {
+                    return false;
+                }
+            }
+
+            throw error;
+        }
+    }
+
+    async delete(path: string): Promise<void> {
+        await this.client.removeObject(this.bucketName, path);
+    }
+
+    async copy(sourcePath: string, destinationPath: string): Promise<void> {
+        await this.client.copyObject(this.bucketName, destinationPath, join(this.bucketName, sourcePath));
+    }
+
+    async move(sourcePath: string, destinationPath: string): Promise<void> {
+        await this.copy(sourcePath, destinationPath);
+        await this.delete(sourcePath);
+    }
+}

src/s3/minio.ts

@@ -3,6 +3,7 @@ import { type IObjectStorage, PutOptions, StatResponse } from "./interface";
 import { parseConnectionString } from "./connectionString";
 import { FileStorage } from "./file/file";
 import { S3Storage } from "./s3/s3";
+import { MinioStorage } from "./s3/minio";
 
 /**
  * The Storage class implements the `IObjectStorage` interface and provides a way to interact
@@ -40,7 +41,11 @@ export class Storage implements IObjectStorage {
                 this.#implementation = new FileStorage(parsedConnectionString.bucketName);
                 break;
             case "s3":
-                this.#implementation = new S3Storage(parsedConnectionString);
+                if (parsedConnectionString.parameters?.useMinioSdk === "true") {
+                    this.#implementation = new MinioStorage(parsedConnectionString);
+                } else {
+                    this.#implementation = new S3Storage(parsedConnectionString);
+                }
                 break;
             // case "azblob":
             //     this.#implementation = new AzureBlobStorage(parsedConnectionString);

src/storage.ts

@@ -1,5 +1,5 @@
 import { afterAll, beforeAll, describe, expect, it } from "vitest";
-import { destroyBucket, removeAllObject, setupBucket } from "./util";
+import { destroyBucket, removeAllObject, setupBucket } from "./aws.util";
 import { S3Client } from "@aws-sdk/client-s3";
 import { loremIpsum } from "lorem-ipsum";
 import { createHash } from "node:crypto";

tests/s3/integration.test.ts renamed to tests/s3/aws.integration.test.ts

@@ -0,0 +1,71 @@
+import { afterAll, beforeAll, describe, expect, it } from "vitest";
+import { destroyBucket, removeAllObject, setupBucket } from "./minio.util";
+import { loremIpsum } from "lorem-ipsum";
+import { createHash } from "node:crypto";
+import { ConnectionString } from "../../src/connectionString";
+import { MinioStorage } from "../../src/s3/minio";
+import { Client } from "minio";
+
+describe("S3 Provider - Integration", () => {
+    const s3Host = process.env.S3_HOST ?? "http://localhost:9000";
+    const s3Access = process.env.S3_ACCESS ?? "teknologi-umum";
+    const s3Secret = process.env.S3_SECRET ?? "very-strong-password";
+    const bucketName = "blob-js";
+    const s3Client = new Client({
+        endPoint: s3Host,
+        accessKey: s3Access,
+        secretKey: s3Secret,
+        useSSL: false,
+        pathStyle: true,
+        region: "us-east-1"
+    });
+
+    const connectionStringConfig: ConnectionString = {
+        provider: "s3",
+        username: s3Access,
+        password: s3Secret,
+        bucketName: bucketName,
+        parameters: {
+            useMinioSdk: "true",
+            endpoint: s3Host,
+            disableHostPrefix: "true",
+            forcePathStyle: "true"
+        }
+    };
+
+    beforeAll(async () => {
+        // Create S3 bucket
+        await setupBucket(s3Client, bucketName);
+    });
+
+    afterAll(async () => {
+        await removeAllObject(s3Client, bucketName);
+        await destroyBucket(s3Client, bucketName);
+    });
+
+    it("should be able to create, read and delete file", async () => {
+        const content = loremIpsum({count: 1024, units: "sentences"});
+        const hashFunc = createHash("md5");
+        hashFunc.update(content);
+        const checksum = hashFunc.digest("base64");
+
+        const s3Client = new MinioStorage(connectionStringConfig);
+
+        await s3Client.put("lorem-ipsum.txt", content, {contentMD5: checksum});
+
+        expect(s3Client.exists("lorem-ipsum.txt"))
+            .resolves
+            .toStrictEqual(true);
+
+        // GetObjectAttributes is not available on MinIO
+        // const fileStat = await s3Client.stat("lorem-ipsum.txt");
+        // expect(fileStat.size).toStrictEqual(content.length);
+
+        const fileContent = await s3Client.get("lorem-ipsum.txt");
+        expect(fileContent.toString()).toStrictEqual(content);
+
+        expect(s3Client.delete("lorem-ipsum.txt"))
+            .resolves
+            .ok;
+    });
+});

tests/s3/util.ts renamed to tests/s3/aws.util.ts

@@ -0,0 +1,38 @@
+import {
+    BucketAlreadyExists,
+    BucketAlreadyOwnedByYou,
+    CreateBucketCommand,
+    DeleteBucketCommand,
+    DeleteObjectsCommand,
+    ListObjectsV2Command,
+    S3Client
+} from "@aws-sdk/client-s3";
+import { Client } from "minio";
+
+export async function setupBucket(client: Client, bucketName: string): Promise<void> {
+    await client.makeBucket(bucketName);
+}
+
+export async function removeAllObject(client: Client, bucketName: string): Promise<void> {
+    const listObj = client.listObjectsV2(bucketName);
+
+    return new Promise((resolve, reject) => {
+        listObj.on("data", async (obj) => {
+            if (obj.name !== undefined) {
+                await client.removeObject(bucketName, obj.name);
+            }
+        });
+    
+        listObj.on("error", (error) => {
+            reject(error);
+        });
+    
+        listObj.on("end", () => {
+            resolve();
+        });
+    });
+}
+
+export async function destroyBucket(client: Client, bucketName: string): Promise<void> {
+    await client.removeBucket(bucketName);
+}

tests/s3/minio.integration.test.ts

@@ -4,7 +4,7 @@ import { createHash } from "node:crypto";
 import { BlobFileNotExistError, BlobMismatchedMD5IntegrityError } from "../../src/errors";
 import { S3Client } from "@aws-sdk/client-s3";
 import { ConnectionString } from "../../src/connectionString";
-import { destroyBucket, removeAllObject, setupBucket } from "./util";
+import { destroyBucket, removeAllObject, setupBucket } from "./aws.util";
 import { S3Storage } from "../../src/s3/s3";
 
 describe("S3 Provider - Unit", () => {