Skip to content

Commit

Permalink
fix new addteters after run
Browse files Browse the repository at this point in the history
  • Loading branch information
Xziy committed Jul 21, 2023
1 parent 598aa31 commit 2563fc1
Show file tree
Hide file tree
Showing 17 changed files with 232 additions and 116 deletions.
1 change: 1 addition & 0 deletions adapters/index.d.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,7 @@ export declare class OTP {
export declare class Adapter {
private static instanceRMS;
private static instanceMF;
static WEBRESTO_MODULES_PATH: string;
static getDiscountAdapter(adapterName?: string, initParams?: {
[key: string]: string | number | boolean;
}): Promise<DiscountAdapter>;
Expand Down
9 changes: 5 additions & 4 deletions adapters/index.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -144,7 +144,7 @@ class Adapter {
if (!adapterName) {
return discountAdapter_1.DiscountAdapter.getInstance();
}
let adapterLocation = WEBRESTO_MODULES_PATH + "/" + adapterName.toLowerCase() + "-discount-adapter";
let adapterLocation = this.WEBRESTO_MODULES_PATH + "/" + adapterName.toLowerCase() + "-discount-adapter";
adapterLocation = fs.existsSync(adapterLocation) ? adapterLocation : "@webresto/" + adapterName.toLowerCase() + "-discount-adapter";
try {
const adapter = require(adapterLocation);
Expand All @@ -164,7 +164,7 @@ class Adapter {
if (!defaultAdapterName)
throw 'BonusProgramAdapter is not set ';
}
let adapterLocation = WEBRESTO_MODULES_PATH + "/" + adapterName.toLowerCase() + "-discount-adapter";
let adapterLocation = this.WEBRESTO_MODULES_PATH + "/" + adapterName.toLowerCase() + "-discount-adapter";
adapterLocation = fs.existsSync(adapterLocation) ? adapterLocation : "@webresto/" + adapterName.toLowerCase() + "-bonus-adapter";
try {
const adapter = require(adapterLocation);
Expand Down Expand Up @@ -201,7 +201,7 @@ class Adapter {
if (!adapterName)
throw 'RMS adapter is not installed';
}
let adapterLocation = WEBRESTO_MODULES_PATH + "/" + adapterName.toLowerCase() + "-rms-adapter";
let adapterLocation = this.WEBRESTO_MODULES_PATH + "/" + adapterName.toLowerCase() + "-rms-adapter";
adapterLocation = fs.existsSync(adapterLocation) ? adapterLocation : "@webresto/" + adapterName.toLowerCase() + "-rms-adapter";
try {
const adapterModule = require(adapterLocation);
Expand Down Expand Up @@ -243,7 +243,7 @@ class Adapter {
}
}
if (!adapterLocation) {
adapterLocation = WEBRESTO_MODULES_PATH + "/" + adapterName.toLowerCase() + "-mediafile-adapter";
adapterLocation = this.WEBRESTO_MODULES_PATH + "/" + adapterName.toLowerCase() + "-mediafile-adapter";
adapterLocation = fs.existsSync(adapterLocation) ? adapterLocation : "@webresto/" + adapterName.toLowerCase() + "-mediafile-adapter";
}
try {
Expand All @@ -257,4 +257,5 @@ class Adapter {
}
}
}
Adapter.WEBRESTO_MODULES_PATH = process.env.WEBRESTO_MODULES_PATH === undefined ? "@webresto" : process.env.WEBRESTO_MODULES_PATH;
exports.Adapter = Adapter;
10 changes: 5 additions & 5 deletions adapters/index.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -162,7 +162,7 @@ export class Adapter {
// Singletons
private static instanceRMS: RMSAdapter;
private static instanceMF: MediaFileAdapter;

public static WEBRESTO_MODULES_PATH = process.env.WEBRESTO_MODULES_PATH === undefined ? "@webresto" : process.env.WEBRESTO_MODULES_PATH;
public static async getDiscountAdapter(adapterName?: string, initParams?: {[key: string]:string | number | boolean}): Promise<DiscountAdapter> {

if(!adapterName) {
Expand All @@ -173,7 +173,7 @@ export class Adapter {
return DiscountAdapter.getInstance();
}

let adapterLocation = WEBRESTO_MODULES_PATH + "/" + adapterName.toLowerCase() + "-discount-adapter";
let adapterLocation = this.WEBRESTO_MODULES_PATH + "/" + adapterName.toLowerCase() + "-discount-adapter";
adapterLocation = fs.existsSync(adapterLocation) ? adapterLocation : "@webresto/" + adapterName.toLowerCase() + "-discount-adapter";

try {
Expand All @@ -194,7 +194,7 @@ export class Adapter {
if (!defaultAdapterName) throw 'BonusProgramAdapter is not set '
}

let adapterLocation = WEBRESTO_MODULES_PATH + "/" + adapterName.toLowerCase() + "-discount-adapter";
let adapterLocation = this.WEBRESTO_MODULES_PATH + "/" + adapterName.toLowerCase() + "-discount-adapter";
adapterLocation = fs.existsSync(adapterLocation) ? adapterLocation : "@webresto/" + adapterName.toLowerCase() + "-bonus-adapter";

try {
Expand Down Expand Up @@ -235,7 +235,7 @@ export class Adapter {
}


let adapterLocation = WEBRESTO_MODULES_PATH + "/" + adapterName.toLowerCase() + "-rms-adapter";
let adapterLocation = this.WEBRESTO_MODULES_PATH + "/" + adapterName.toLowerCase() + "-rms-adapter";
adapterLocation = fs.existsSync(adapterLocation) ? adapterLocation : "@webresto/" + adapterName.toLowerCase() + "-rms-adapter";

try {
Expand Down Expand Up @@ -281,7 +281,7 @@ export class Adapter {
}

if (!adapterLocation) {
adapterLocation = WEBRESTO_MODULES_PATH + "/" + adapterName.toLowerCase() + "-mediafile-adapter";
adapterLocation = this.WEBRESTO_MODULES_PATH + "/" + adapterName.toLowerCase() + "-mediafile-adapter";
adapterLocation = fs.existsSync(adapterLocation) ? adapterLocation : "@webresto/" + adapterName.toLowerCase() + "-mediafile-adapter";
}

Expand Down
2 changes: 1 addition & 1 deletion adapters/mediafile/MediaFileAdapter.d.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ export default abstract class MediaFileAdapter {
*/
wait(): Promise<void>;
toDownload(url: string, target: string, type: MediaFileTypes, force?: boolean): Promise<MediaFile>;
abstract load(url: string, type: MediaFileTypes, config: BaseConfigProperty): Promise<{
abstract process(url: string, type: MediaFileTypes, config: BaseConfigProperty): Promise<{
origin: string;
small: string;
large: string;
Expand Down
3 changes: 2 additions & 1 deletion adapters/mediafile/MediaFileAdapter.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ class MediaFileAdapter {
}
async toDownload(url, target, type, force = false) {
await this.wait();
sails.log.silly(`Adapter > Mediafile > toDownload: ${url}`);
let imageId = (0, uuid_1.v5)(url, this.UUID_NAMESPACE);
let mediaFile = await MediaFile.findOne({ id: imageId });
let loadConfig;
Expand All @@ -34,7 +35,7 @@ class MediaFileAdapter {
};
switch (type) {
case "image":
mediaFile.images = this.load(url, "image", loadConfig);
mediaFile.images = this.process(url, "image", loadConfig);
break;
case "video":
// mediaFile.video = ???
Expand Down
5 changes: 3 additions & 2 deletions adapters/mediafile/MediaFileAdapter.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@ export default abstract class MediaFileAdapter {

public async toDownload(url: string, target: string, type: MediaFileTypes, force: boolean = false): Promise<MediaFile> {
await this.wait()
sails.log.silly(`Adapter > Mediafile > toDownload: ${url}`)
let imageId = uuidv5(url, this.UUID_NAMESPACE);
let mediaFile = await MediaFile.findOne({ id: imageId });

Expand All @@ -56,7 +57,7 @@ export default abstract class MediaFileAdapter {
switch (type)
{
case "image":
mediaFile.images = this.load(url, "image", loadConfig);
mediaFile.images = this.process(url, "image", loadConfig);
break;

case "video":
Expand All @@ -78,5 +79,5 @@ export default abstract class MediaFileAdapter {
};


public abstract load(url: string, type: MediaFileTypes, config: BaseConfigProperty): Promise<{ origin: string, small: string, large: string }>;
public abstract process(url: string, type: MediaFileTypes, config: BaseConfigProperty): Promise<{ origin: string, small: string, large: string }>;
}
5 changes: 4 additions & 1 deletion adapters/mediafile/default/local.d.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,14 +26,17 @@ interface LoadMediaFilesProcess {
config: MediaFileConfigInner;
}
export default class LocalMediaFileAdapter extends MediaFileAdapter {
private processing;
private processingTimeout;
loadMediaFilesProcessQueue: LoadMediaFilesProcess[];
constructor(config: BaseConfig);
getNameByUrl(url: string, ext: string, options?: any, salt?: boolean): string;
load(url: string, type: MediaFileTypes, config: BaseConfig): Promise<{
process(url: string, type: MediaFileTypes, config: BaseConfig): Promise<{
origin: string;
small: string;
large: string;
}>;
protected getPrefix(type: string): string;
protected download(loadMediaFilesProcess: LoadMediaFilesProcess): Promise<void>;
private loadMediaFiles;
}
Expand Down
123 changes: 83 additions & 40 deletions adapters/mediafile/default/local.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ const path = require("path");
class LocalMediaFileAdapter extends MediaFileAdapter_1.default {
constructor(config) {
super(config);
this.processing = false;
this.loadMediaFilesProcessQueue = [];
this.loadMediaFiles();
}
Expand All @@ -39,10 +40,10 @@ class LocalMediaFileAdapter extends MediaFileAdapter_1.default {
baseName = (0, uuid_1.v5)(baseName, this.UUID_NAMESPACE);
if (salt)
baseName += `-${(new Date()).getTime()}`;
baseName += ext;
baseName += `.${ext}`;
return baseName;
}
async load(url, type, config) {
async process(url, type, config) {
const baseConfig = {
format: "webp",
resize: {
Expand All @@ -58,7 +59,7 @@ class LocalMediaFileAdapter extends MediaFileAdapter_1.default {
background: "white"
};
const cfg = { ...baseConfig, ...config };
const mediafileExtesion = url.match(/\.([0-9a-z]+)(?=[?#])|(\.)(?:[\w]+)$/gim)[0];
const mediafileExtesion = url.match(/\.([0-9a-z]+)(?=[?#])|(\.)(?:[\w]+)$/gim)[0].replace('.', '');
const origin = this.getNameByUrl(url, mediafileExtesion);
const name = {
origin: origin,
Expand All @@ -76,52 +77,94 @@ class LocalMediaFileAdapter extends MediaFileAdapter_1.default {
});
return name;
}
getPrefix(type) {
return path.join(process.cwd(), ".tmp/public", type);
}
async download(loadMediaFilesProcess) {
const prefix = path.join(process.cwd(), ".tmp/public", loadMediaFilesProcess.type);
const prefix = this.getPrefix(loadMediaFilesProcess.type);
const fullPathDl = path.join(prefix, loadMediaFilesProcess.name.origin);
const response = await axios_1.default.get(loadMediaFilesProcess.url, { responseType: 'stream' });
fs.mkdirSync(prefix, { recursive: true });
const writer = fs.createWriteStream(fullPathDl);
response.data.pipe(writer);
await new Promise((resolve, reject) => {
writer.on('finish', resolve);
writer.on('error', reject);
});
// Check if file exists
if (!fs.existsSync(fullPathDl)) {
const response = await axios_1.default.get(loadMediaFilesProcess.url, { responseType: 'stream' });
sails.log.silly(`MF local > download image: ${fullPathDl}, status: ${response.status}`);
fs.mkdirSync(prefix, { recursive: true });
const writer = fs.createWriteStream(fullPathDl);
response.data.pipe(writer);
await new Promise((resolve, reject) => {
writer.on('finish', resolve);
writer.on('error', reject);
});
}
else {
sails.log.silly(`File ${fullPathDl} already exists. Skipping download.`);
}
}
async loadMediaFiles() {
const MEDIAFILE_PARALEL_TO_DOWNLOAD = await Settings.get('MEDIAFILE_PARALEL_TO_DOWNLOAD') ?? 3;
if (this.processing) {
return;
}
if (this.loadMediaFilesProcessQueue.length === 0) {
sails.log.silly(`MF local > no mediafiles to download`);
this.processingTimeout = setTimeout(this.loadMediaFiles.bind(this), 30000);
return;
}
this.processing = true;
let MEDIAFILE_PARALEL_TO_DOWNLOAD = await Settings.get('MEDIAFILE_PARALEL_TO_DOWNLOAD') ?? 3;
if (MEDIAFILE_PARALEL_TO_DOWNLOAD > this.loadMediaFilesProcessQueue.length)
MEDIAFILE_PARALEL_TO_DOWNLOAD = this.loadMediaFilesProcessQueue.length;
while (this.loadMediaFilesProcessQueue.length) {
const loadMediaFilesProcesses = this.loadMediaFilesProcessQueue.splice(0, MEDIAFILE_PARALEL_TO_DOWNLOAD);
const downloadPromises = loadMediaFilesProcesses.map(async (loadMediaFilesProcess) => {
const prefix = await this.download(loadMediaFilesProcess);
switch (loadMediaFilesProcess.type) {
case "image":
for (let size in loadMediaFilesProcess.config.resize) {
if (size === "origin")
continue;
const mediafileItem = loadMediaFilesProcess.config.resize[size];
if (!mediafileItem.width && !mediafileItem.height) {
throw "Not valid mediafile config. Must have name (key) and one of width or height";
console.log(this.loadMediaFilesProcessQueue.length, 10000);
const downloadPromises = loadMediaFilesProcesses.map((loadMediaFilesProcess) => {
return this.download(loadMediaFilesProcess).then(async () => {
const prefix = this.getPrefix(loadMediaFilesProcess.type);
switch (loadMediaFilesProcess.type) {
case "image":
sails.log.silly(`MF local > process image: ${loadMediaFilesProcess.name.origin}`);
for (let size in loadMediaFilesProcess.config.resize) {
const dstPath = path.join(prefix, loadMediaFilesProcess.name[size]);
if (!fs.existsSync(dstPath)) {
if (size === "origin")
continue;
const mediafileItem = loadMediaFilesProcess.config.resize[size];
if (!mediafileItem.width && !mediafileItem.height) {
throw "Not valid mediafile config. Must have name (key) and one of width or height";
}
mediafileItem.width = mediafileItem.width || mediafileItem.height;
mediafileItem.height = mediafileItem.height || mediafileItem.width;
await resizeMediaFile({
srcPath: path.join(prefix, loadMediaFilesProcess.name.origin),
dstPath: path.join(prefix, loadMediaFilesProcess.name[size]),
width: mediafileItem.width,
height: mediafileItem.height,
customArgs: ["-background", loadMediaFilesProcess.config.background || "white", "-flatten"],
});
sails.log.silly(`MF local > process finished: ${loadMediaFilesProcess.name[size]}`);
}
else {
sails.log.silly(`MF local > process skip existing processed file: ${loadMediaFilesProcess.name[size]}`);
}
}
mediafileItem.width = mediafileItem.width || mediafileItem.height;
mediafileItem.height = mediafileItem.height || mediafileItem.width;
await resizeMediaFile({
srcPath: path.normalize(prefix + loadMediaFilesProcess.name.origin),
dstPath: path.normalize(prefix + loadMediaFilesProcess.name[size]),
width: mediafileItem.width,
height: mediafileItem.height,
customArgs: ["-background", loadMediaFilesProcess.config.background || "white", "-flatten"],
});
}
break;
default:
break;
}
break;
default:
break;
}
}).catch(error => {
// Log the error and rethrow it
sails.log.error(`Error processing file ${loadMediaFilesProcess.name.origin}: ${error}`);
});
});
// Wait for all downloads and processing to complete
await Promise.all(downloadPromises);
try {
// Wait for all downloads and processing to complete
await Promise.all(downloadPromises);
}
catch (error) {
// Handle errors that occurred during processing
sails.log.error(`An error occurred during file processing: ${error}`);
}
}
setTimeout(this.loadMediaFiles.bind(this), 10000);
this.processing = false;
this.processingTimeout = setTimeout(this.loadMediaFiles.bind(this), 30000);
}
}
exports.default = LocalMediaFileAdapter;
Expand Down
Loading

0 comments on commit 2563fc1

Please sign in to comment.