From 138e65a6958f9e1dc61b62c9444307511cf42d57 Mon Sep 17 00:00:00 2001 From: Xziy Date: Fri, 21 Jul 2023 04:29:40 +0300 Subject: [PATCH] update MF --- adapters/index.js | 5 +- adapters/index.ts | 4 +- adapters/mediafile/MediaFileAdapter.js | 9 ++-- adapters/mediafile/MediaFileAdapter.ts | 12 +++-- adapters/mediafile/default/local.ts | 67 ++++++++++++++------------ adapters/rms/RMSAdapter.js | 15 ++++-- adapters/rms/RMSAdapter.ts | 14 ++++-- 7 files changed, 76 insertions(+), 50 deletions(-) diff --git a/adapters/index.js b/adapters/index.js index 0db2f13c..b8841087 100644 --- a/adapters/index.js +++ b/adapters/index.js @@ -4,10 +4,10 @@ exports.Adapter = exports.OTP = exports.Captcha = exports.Payment = exports.Map const RMSAdapter_1 = require("./rms/RMSAdapter"); const pow_1 = require("./captcha/default/pow"); const defaultOTP_1 = require("./otp/default/defaultOTP"); +const local_1 = require("./mediafile/default/local"); const MediaFileAdapter_1 = require("./mediafile/MediaFileAdapter"); const fs = require("fs"); const discountAdapter_1 = require("./discount/default/discountAdapter"); -const path = require("path"); // import DiscountAdapter from "./discount/AbstractDiscountAdapter"; const WEBRESTO_MODULES_PATH = process.env.WEBRESTO_MODULES_PATH === undefined ? "@webresto" : process.env.WEBRESTO_MODULES_PATH; /** @@ -238,7 +238,8 @@ class Adapter { if (!adapterName) { adapterName = await Settings.get("DEFAULT_MEDIAFILE_ADAPTER"); if (!adapterName) { - adapterLocation = path.resolve(__dirname, "mediafile/default/local"); + this.instanceMF = new local_1.default(initParams); + return this.instanceMF; } } if (!adapterLocation) { diff --git a/adapters/index.ts b/adapters/index.ts index 3c75a0a8..b979e215 100644 --- a/adapters/index.ts +++ b/adapters/index.ts @@ -3,6 +3,7 @@ import MapAdapter from "./map/MapAdapter"; import CaptchaAdapter from "./captcha/CaptchaAdapter"; import { POW } from "./captcha/default/pow"; import { DefaultOTP } from "./otp/default/defaultOTP"; +import LocalMediaFileAdapter from "./mediafile/default/local"; import OTPAdapter from "./otp/OneTimePasswordAdapter" import MediaFileAdapter, { ConfigMediaFileAdapter } from "./mediafile/MediaFileAdapter"; import PaymentAdapter from "./payment/PaymentAdapter"; @@ -274,7 +275,8 @@ export class Adapter { if(!adapterName) { adapterName = await Settings.get("DEFAULT_MEDIAFILE_ADAPTER") as string; if (!adapterName) { - adapterLocation = path.resolve(__dirname, "mediafile/default/local") + this.instanceMF = new LocalMediaFileAdapter(initParams); + return this.instanceMF; } } diff --git a/adapters/mediafile/MediaFileAdapter.js b/adapters/mediafile/MediaFileAdapter.js index 04d53172..c975dc7a 100644 --- a/adapters/mediafile/MediaFileAdapter.js +++ b/adapters/mediafile/MediaFileAdapter.js @@ -22,13 +22,16 @@ class MediaFileAdapter { async toDownload(url, target, type, force = false) { await this.wait(); let imageId = (0, uuid_1.v5)(url, this.UUID_NAMESPACE); - const mediaFile = await MediaFile.findOne({ id: imageId }); + let mediaFile = await MediaFile.findOne({ id: imageId }); let loadConfig; - if (target && this.config[target]) { + if (target && this.config && this.config[target]) { loadConfig = this.config[target]; } // image if (mediaFile === undefined || force) { + mediaFile = { + id: imageId + }; switch (type) { case "image": mediaFile.images = this.load(url, "image", loadConfig); @@ -43,7 +46,7 @@ class MediaFileAdapter { throw `mediaFile type not known ${type}`; break; } - // DOWNLOAD + mediaFile = await MediaFile.create(mediaFile).fetch(); } return mediaFile; } diff --git a/adapters/mediafile/MediaFileAdapter.ts b/adapters/mediafile/MediaFileAdapter.ts index 58718bd8..917e7e06 100644 --- a/adapters/mediafile/MediaFileAdapter.ts +++ b/adapters/mediafile/MediaFileAdapter.ts @@ -40,16 +40,19 @@ export default abstract class MediaFileAdapter { public async toDownload(url: string, target: string, type: MediaFileTypes, force: boolean = false): Promise { await this.wait() let imageId = uuidv5(url, this.UUID_NAMESPACE); - const mediaFile = await MediaFile.findOne({ id: imageId }); + let mediaFile = await MediaFile.findOne({ id: imageId }); let loadConfig: BaseConfigProperty; - if (target && this.config[target]){ + if (target && this.config && this.config[target]){ loadConfig = this.config[target]; } // image if (mediaFile === undefined || force) { + mediaFile = { + id: imageId + }; switch (type) { case "image": @@ -68,10 +71,9 @@ export default abstract class MediaFileAdapter { throw `mediaFile type not known ${type}` break; } - - // DOWNLOAD + mediaFile = await MediaFile.create(mediaFile).fetch() } - + return mediaFile; }; diff --git a/adapters/mediafile/default/local.ts b/adapters/mediafile/default/local.ts index 4ad45962..a8a1eaf2 100644 --- a/adapters/mediafile/default/local.ts +++ b/adapters/mediafile/default/local.ts @@ -134,39 +134,44 @@ export default class LocalMediaFileAdapter extends MediaFileAdapter { } private async loadMediaFiles() { - while (this.loadMediaFilesProcessQueue.length) { - const loadMediaFilesProcess = this.loadMediaFilesProcessQueue.shift(); + const MEDIAFILE_PARALEL_TO_DOWNLOAD = await Settings.get('MEDIAFILE_PARALEL_TO_DOWNLOAD') as number ?? 3; + + while (this.loadMediaFilesProcessQueue.length) { + const loadMediaFilesProcesses = this.loadMediaFilesProcessQueue.splice(0, MEDIAFILE_PARALEL_TO_DOWNLOAD); + + const downloadPromises = loadMediaFilesProcesses.map(async (loadMediaFilesProcess) => { + const prefix = await this.download(loadMediaFilesProcess); + + switch (loadMediaFilesProcess.type) { + case "image": + for (let size in loadMediaFilesProcess.config.resize) { + if (size === "origin") continue; + const mediafileItem = loadMediaFilesProcess.config.resize[size]; + if (!mediafileItem.width && !mediafileItem.height) { + throw "Not valid mediafile config. Must have name (key) and one of width or height"; + } + mediafileItem.width = mediafileItem.width || mediafileItem.height; + mediafileItem.height = mediafileItem.height || mediafileItem.width; + + await resizeMediaFile({ + srcPath: path.normalize(prefix + loadMediaFilesProcess.name.origin), + dstPath: path.normalize(prefix + loadMediaFilesProcess.name[size]), + width: mediafileItem.width, + height: mediafileItem.height, + customArgs: ["-background", loadMediaFilesProcess.config.background || "white", "-flatten"], + }); + } + break; + default: + break; + } + }); - const prefix = await this.download(loadMediaFilesProcess); + // Wait for all downloads and processing to complete + await Promise.all(downloadPromises); + } - switch (loadMediaFilesProcess.type) - { - case "image": - for (let size in loadMediaFilesProcess.config.resize) { - if (size === "origin") continue; - - const mediafileItem = loadMediaFilesProcess.config.resize[size]; - if (!mediafileItem.width && !mediafileItem.height) { - throw "Not valid mediafile config. Must have name (key) and one of width or height"; - } - - mediafileItem.width = mediafileItem.width || mediafileItem.height; - mediafileItem.height = mediafileItem.height || mediafileItem.width; - - await resizeMediaFile({ - srcPath: path.normalize(prefix + loadMediaFilesProcess.name.origin), - dstPath: path.normalize(prefix + loadMediaFilesProcess.name[size]), - width: mediafileItem.width, - height: mediafileItem.height, - customArgs: ["-background", loadMediaFilesProcess.config.background || "white", "-flatten"], - }); - } - break; - default: - break; - } - } - setTimeout(this.loadMediaFiles.bind(this), 60000); + setTimeout(this.loadMediaFiles.bind(this), 10000); } } diff --git a/adapters/rms/RMSAdapter.js b/adapters/rms/RMSAdapter.js index 515f58cc..ac971f29 100644 --- a/adapters/rms/RMSAdapter.js +++ b/adapters/rms/RMSAdapter.js @@ -54,7 +54,11 @@ class RMSAdapter { return; } this.syncProductsExecuted = true; - const rootGroupsToSync = await Settings.get("ROOT_GROUPS_RMS_TO_SYNC"); + let rootGroupsToSync = await Settings.get("ROOT_GROUPS_RMS_TO_SYNC"); + if (typeof rootGroupsToSync === "string") + rootGroupsToSync = rootGroupsToSync.split(";"); + if (!rootGroupsToSync) + rootGroupsToSync = []; const rmsAdapter = await Adapter.getRMSAdapter(); if (await rmsAdapter.nomenclatureHasUpdated() || force) { const currentRMSGroupsFlatTree = await rmsAdapter.loadNomenclatureTree(rootGroupsToSync); @@ -77,14 +81,17 @@ class RMSAdapter { allProductIds = allProductIds.concat(productIds); for (let product of productsToUpdate) { emitter.emit("rms-sync:before-each-product-item", product); + const SKIP_LOAD_PRODUCT_IMAGES = await Settings.get("SKIP_LOAD_PRODUCT_IMAGES") ?? false; // Load images - if (product.images && product.images.length) { + if (product.images && product.images.length && !SKIP_LOAD_PRODUCT_IMAGES) { + console.log(product.images); const isURL = (str) => /^(https?:\/\/)?[\w.-]+(\.[\w.-]+)+[\w\-._~:/?#[\]@!$&'()*+,;=]+$/.test(str); for (let image of product.images) { if (isURL(image)) { // load image - let mfAdater = await Adapter.getMediaFileAdapter(); - let mediaFileImage = await mfAdater.toDownload(image, 'dish', 'image'); + const mfAdater = await Adapter.getMediaFileAdapter(); + const mediaFileImage = await mfAdater.toDownload(image, 'dish', 'image'); + console.log(mediaFileImage); await Dish.addToCollection(product.id, 'images').members([mediaFileImage.id]); } else { diff --git a/adapters/rms/RMSAdapter.ts b/adapters/rms/RMSAdapter.ts index 0fd68a5b..5748005a 100644 --- a/adapters/rms/RMSAdapter.ts +++ b/adapters/rms/RMSAdapter.ts @@ -90,7 +90,10 @@ export default abstract class RMSAdapter { } this.syncProductsExecuted = true; - const rootGroupsToSync = await Settings.get("ROOT_GROUPS_RMS_TO_SYNC") as string[]; + let rootGroupsToSync = await Settings.get("ROOT_GROUPS_RMS_TO_SYNC") as string | string[]; + if(typeof rootGroupsToSync === "string") rootGroupsToSync = rootGroupsToSync.split(";"); + if (!rootGroupsToSync) rootGroupsToSync = []; + const rmsAdapter = await Adapter.getRMSAdapter(); if (await rmsAdapter.nomenclatureHasUpdated() || force) { @@ -123,14 +126,17 @@ export default abstract class RMSAdapter { for (let product of productsToUpdate) { emitter.emit("rms-sync:before-each-product-item", product); + const SKIP_LOAD_PRODUCT_IMAGES = await Settings.get("SKIP_LOAD_PRODUCT_IMAGES") as boolean ?? false // Load images - if(product.images && product.images.length) { + if(product.images && product.images.length && !SKIP_LOAD_PRODUCT_IMAGES) { + console.log(product.images) const isURL = (str) => /^(https?:\/\/)?[\w.-]+(\.[\w.-]+)+[\w\-._~:/?#[\]@!$&'()*+,;=]+$/.test(str); for (let image of product.images){ if (isURL(image)){ // load image - let mfAdater = await Adapter.getMediaFileAdapter() - let mediaFileImage = await mfAdater.toDownload(image as string, 'dish', 'image'); + const mfAdater = await Adapter.getMediaFileAdapter() + const mediaFileImage = await mfAdater.toDownload(image as string, 'dish', 'image'); + console.log(mediaFileImage) await Dish.addToCollection(product.id, 'images').members([mediaFileImage.id]); } else { sails.log.debug(`Image not url on sync products ${image}`);