Skip to content

Commit 0320a1a

Browse files
foxriver76foxriver76
authored
[fix]: fixed iob setup custom migration and iob validate (#2951)
* fixed 'iob validate' and 'setup custom' migration * fix restore process for restore on migration * add test * make the backup name not a number on CI * another try * do not log backup ok if it failed * fix backups missing postfix * another try
1 parent 1ee989d commit 0320a1a

File tree

4 files changed

+111
-80
lines changed

4 files changed

+111
-80
lines changed

‎CHANGELOG.md‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,9 @@
44
## __WORK IN PROGRESS__
55
-->
66

7+
## __WORK IN PROGRESS__
8+
* (@foxriver76) fixed `iob validate` command and `setup custom` migration
9+
710
## 7.0.1 (2024-10-21) - Lucy
811
* (@foxriver76) fixed crash case on database migration
912
* (@foxriver76) fixed edge case crash cases if notifications are processed nearly simultaneously

‎packages/cli/src/lib/setup.ts‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1328,9 +1328,9 @@ async function processCommand(
13281328
await backup.validateBackup(name);
13291329
console.log('Backup OK');
13301330
return void exitApplicationSave(0);
1331-
} catch (err) {
1332-
console.log(`Backup check failed: ${err.message}`);
1333-
return void exitApplicationSave(1);
1331+
} catch (e) {
1332+
console.log(`Backup check failed: ${e.message}`);
1333+
return void exitApplicationSave(e instanceof IoBrokerError ? e.code : 1);
13341334
}
13351335
});
13361336
break;

‎packages/cli/src/lib/setup/setupBackup.ts‎

Lines changed: 97 additions & 75 deletions
Original file line numberDiff line numberDiff line change
@@ -90,6 +90,8 @@ export class BackupRestore {
9090
private readonly HOSTNAME_PLACEHOLDER_REPLACE = '$$$$__hostname__$$$$';
9191
/** Regex to replace all occurrences of the HOSTNAME_PLACEHOLDER */
9292
private readonly HOSTNAME_PLACEHOLDER_REGEX = /\$\$__hostname__\$\$/g;
93+
/** Postfix for backup name */
94+
private readonly BACKUP_POSTFIX = `_backup${tools.appNameLowerCase}`;
9395

9496
constructor(options: CLIBackupRestoreOptions) {
9597
options = options || {};
@@ -261,7 +263,9 @@ export class BackupRestore {
261263
const d = new Date();
262264
name = `${d.getFullYear()}_${`0${d.getMonth() + 1}`.slice(-2)}_${`0${d.getDate()}`.slice(-2)}-${`0${d.getHours()}`.slice(
263265
-2,
264-
)}_${`0${d.getMinutes()}`.slice(-2)}_${`0${d.getSeconds()}`.slice(-2)}_backup${tools.appName}`;
266+
)}_${`0${d.getMinutes()}`.slice(-2)}_${`0${d.getSeconds()}`.slice(-2)}${this.BACKUP_POSTFIX}`;
267+
} else if (!name.endsWith(this.BACKUP_POSTFIX) && !name.endsWith(`${this.BACKUP_POSTFIX}.tar.gz`)) {
268+
name += this.BACKUP_POSTFIX;
265269
}
266270

267271
name = name.toString().replace(/\\/g, '/');
@@ -367,7 +371,7 @@ export class BackupRestore {
367371

368372
console.log(`host.${hostname} Validating backup ...`);
369373
try {
370-
await this._validateBackupAfterCreation(noConfig);
374+
await this._validateTempDirectory(noConfig);
371375
console.log(`host.${hostname} The backup is valid!`);
372376

373377
return await this._packBackup(name);
@@ -658,8 +662,14 @@ export class BackupRestore {
658662

659663
const backupBaseDir = path.join(this.tmpDir, 'backup');
660664

661-
const config: ioBroker.IoBrokerJson = await fs.readJSON(path.join(backupBaseDir, 'config.json'));
662-
const backupHostName = config.system?.hostname || hostname;
665+
let backupHostName = hostname;
666+
// Note: on backups created during migration no config exists
667+
let config: ioBroker.IoBrokerJson | undefined;
668+
669+
if (await fs.pathExists(path.join(backupBaseDir, 'config.json'))) {
670+
config = (await fs.readJSON(path.join(backupBaseDir, 'config.json'))) as ioBroker.IoBrokerJson;
671+
backupHostName = config.system?.hostname || hostname;
672+
}
663673

664674
// we need to find the host obj for the compatibility check
665675
const objFd = await open(path.join(backupBaseDir, 'objects.jsonl'));
@@ -692,8 +702,10 @@ export class BackupRestore {
692702
}
693703

694704
// restore ioBroker.json
695-
fs.writeFileSync(tools.getConfigFileName(), JSON.stringify(config, null, 2));
696-
await this.connectToNewDatabase(config);
705+
if (config) {
706+
fs.writeFileSync(tools.getConfigFileName(), JSON.stringify(config, null, 2));
707+
await this.connectToNewDatabase(config);
708+
}
697709

698710
console.log(`host.${hostname} Clear all objects and states...`);
699711
await this.cleanDatabase(false);
@@ -746,7 +758,7 @@ export class BackupRestore {
746758
const { force, restartOnFinish, dontDeleteAdapters } = options;
747759

748760
const backupBaseDir = path.join(this.tmpDir, 'backup');
749-
const isJsonl = await fs.pathExists(path.join(backupBaseDir, 'config.json'));
761+
const isJsonl = await fs.pathExists(path.join(backupBaseDir, 'objects.jsonl'));
750762

751763
if (isJsonl) {
752764
const exitCode = await this._restoreJsonlBackup(options);
@@ -966,27 +978,34 @@ export class BackupRestore {
966978
}
967979

968980
/**
969-
* Validates the backup.json and all json files inside the backup after (in temporary directory), here we only abort if backup.json is corrupted
981+
* Validates a JSONL-style backup and all json files inside the backup (in temporary directory)
970982
*
971983
* @param noConfig if the backup does not contain a `config.json` (used by setup custom migration)
972984
*/
973-
private async _validateBackupAfterCreation(noConfig = false): Promise<void> {
985+
private async _validateTempDirectory(noConfig = false): Promise<void> {
974986
const backupBaseDir = path.join(this.tmpDir, 'backup');
975987

976988
if (!noConfig) {
977989
await fs.readJSON(path.join(backupBaseDir, 'config.json'));
990+
console.log(`host.${this.hostname} "config.json" is valid`);
978991
}
979992

980993
if (!(await fs.pathExists(path.join(backupBaseDir, 'objects.jsonl')))) {
981994
throw new Error('Backup does not contain valid objects');
982995
}
983996

997+
console.log(`host.${this.hostname} "objects.jsonl" exists`);
998+
984999
if (!(await fs.pathExists(path.join(backupBaseDir, 'states.jsonl')))) {
9851000
throw new Error('Backup does not contain valid states');
9861001
}
9871002

1003+
console.log(`host.${this.hostname} "states.jsonl" exists`);
1004+
9881005
await this._validateDatabaseFiles();
9891006

1007+
console.log(`host.${this.hostname} JSONL lines are valid`);
1008+
9901009
// we check all other json files, we assume them as optional, because user created files may be no valid json
9911010
try {
9921011
this._checkDirectory(path.join(backupBaseDir, 'files'));
@@ -1034,7 +1053,7 @@ export class BackupRestore {
10341053
*
10351054
* @param _name - index or name of the backup
10361055
*/
1037-
validateBackup(_name: string | number): Promise<void> | undefined {
1056+
async validateBackup(_name: string | number): Promise<void> {
10381057
let backups;
10391058
let name = typeof _name === 'number' ? _name.toString() : _name;
10401059

@@ -1046,12 +1065,12 @@ export class BackupRestore {
10461065
console.log('Please specify one of the backup names:');
10471066

10481067
for (const t in backups) {
1049-
console.log(`${backups[t]} or ${backups[t].replace(`_backup${tools.appName}.tar.gz`, '')} or ${t}`);
1068+
console.log(`${backups[t]} or ${backups[t].replace(`${this.BACKUP_POSTFIX}.tar.gz`, '')} or ${t}`);
10501069
}
10511070
} else {
10521071
console.warn(`No backups found. Create a backup, using "${tools.appName} backup" first`);
10531072
}
1054-
return void this.processExit(EXIT_CODES.INVALID_ARGUMENTS);
1073+
throw new IoBrokerError({ message: 'Backup not found', code: EXIT_CODES.INVALID_ARGUMENTS });
10551074
}
10561075
// If number
10571076
if (parseInt(name, 10).toString() === name.toString()) {
@@ -1064,95 +1083,98 @@ export class BackupRestore {
10641083
console.log('Please specify one of the backup names:');
10651084
for (const t in backups) {
10661085
console.log(
1067-
`${backups[t]} or ${backups[t].replace(`_backup${tools.appName}.tar.gz`, '')} or ${t}`,
1086+
`${backups[t]} or ${backups[t].replace(`${this.BACKUP_POSTFIX}.tar.gz`, '')} or ${t}`,
10681087
);
10691088
}
10701089
} else {
10711090
console.log(`No existing backups. Create a backup, using "${tools.appName} backup" first`);
10721091
}
1073-
return void this.processExit(EXIT_CODES.INVALID_ARGUMENTS);
1092+
1093+
throw new IoBrokerError({ message: 'Backup not found', code: EXIT_CODES.INVALID_ARGUMENTS });
10741094
}
10751095
console.log(`host.${this.hostname} Using backup file ${name}`);
10761096
}
10771097

10781098
name = name.toString().replace(/\\/g, '/');
10791099
if (!name.includes('/')) {
10801100
name = BackupRestore.getBackupDir() + name;
1081-
const regEx = new RegExp(`_backup${tools.appName}`, 'i');
1101+
const regEx = new RegExp(this.BACKUP_POSTFIX, 'i');
10821102
if (!regEx.test(name)) {
1083-
name += `_backup${tools.appName}`;
1103+
name += this.BACKUP_POSTFIX;
10841104
}
10851105
if (!name.match(/\.tar\.gz$/i)) {
10861106
name += '.tar.gz';
10871107
}
10881108
}
10891109
if (!fs.existsSync(name)) {
10901110
console.error(`host.${this.hostname} Cannot find ${name}`);
1091-
return void this.processExit(EXIT_CODES.INVALID_ARGUMENTS);
1111+
throw new IoBrokerError({ message: 'Backup not found', code: EXIT_CODES.INVALID_ARGUMENTS });
10921112
}
10931113

10941114
if (fs.existsSync(`${this.tmpDir}/backup/backup.json`)) {
10951115
fs.unlinkSync(`${this.tmpDir}/backup/backup.json`);
10961116
}
10971117

1098-
return new Promise(resolve => {
1099-
tar.extract(
1100-
{
1101-
file: name,
1102-
cwd: this.tmpDir,
1103-
},
1104-
undefined,
1105-
err => {
1106-
if (err) {
1107-
console.error(`host.${this.hostname} Cannot extract from file "${name}": ${err.message}`);
1108-
return void this.processExit(EXIT_CODES.INVALID_ARGUMENTS);
1109-
}
1110-
if (!fs.existsSync(`${this.tmpDir}/backup/backup.json`)) {
1111-
console.error(
1112-
`host.${this.hostname} Validation failed. Cannot find extracted file from file "${this.tmpDir}/backup/backup.json"`,
1113-
);
1114-
return void this.processExit(EXIT_CODES.CANNOT_EXTRACT_FROM_ZIP);
1115-
}
1118+
try {
1119+
await tar.extract({
1120+
file: name,
1121+
cwd: this.tmpDir,
1122+
});
1123+
} catch (e) {
1124+
const errMessage = `Cannot extract from file "${name}": ${e.message}`;
1125+
console.error(`host.${this.hostname} ${errMessage}`);
1126+
throw new IoBrokerError({ message: 'Backup not found', code: EXIT_CODES.INVALID_ARGUMENTS });
1127+
}
11161128

1117-
console.log(`host.${this.hostname} Starting validation ...`);
1118-
let backupJSON;
1119-
try {
1120-
backupJSON = fs.readJSONSync(`${this.tmpDir}/backup/backup.json`);
1121-
} catch (err) {
1122-
console.error(
1123-
`host.${this.hostname} Backup corrupted. Backup ${name} does not contain a valid backup.json file: ${err.message}`,
1124-
);
1125-
this.removeTempBackupDir();
1129+
try {
1130+
if (fs.existsSync(path.join(this.tmpDir, 'backup', 'backup.json'))) {
1131+
this._validateLegacyTempDir();
1132+
} else {
1133+
await this._validateTempDirectory();
1134+
}
1135+
} catch (e) {
1136+
console.error(`host.${this.hostname} ${e.message}`);
11261137

1127-
return void this.processExit(EXIT_CODES.CANNOT_EXTRACT_FROM_ZIP);
1128-
}
1138+
try {
1139+
this.removeTempBackupDir();
1140+
} catch (e) {
1141+
console.error(`host.${this.hostname} Cannot clear temporary backup directory: ${e.message}`);
1142+
}
11291143

1130-
if (!backupJSON || !backupJSON.objects || !backupJSON.objects.length) {
1131-
console.error(`host.${this.hostname} Backup corrupted. Backup does not contain valid objects`);
1132-
try {
1133-
this.removeTempBackupDir();
1134-
} catch (e) {
1135-
console.error(
1136-
`host.${this.hostname} Cannot clear temporary backup directory: ${e.message}`,
1137-
);
1138-
}
1139-
return void this.processExit(EXIT_CODES.CANNOT_EXTRACT_FROM_ZIP);
1140-
}
1144+
throw new IoBrokerError({ message: e.message, code: EXIT_CODES.CANNOT_EXTRACT_FROM_ZIP });
1145+
}
11411146

1142-
console.log(`host.${this.hostname} backup.json OK`);
1147+
try {
1148+
this.removeTempBackupDir();
1149+
} catch (e) {
1150+
console.error(`host.${this.hostname} Cannot clear temporary backup directory: ${e.message}`);
1151+
throw new IoBrokerError({ message: e.message, code: EXIT_CODES.CANNOT_EXTRACT_FROM_ZIP });
1152+
}
1153+
}
11431154

1144-
try {
1145-
this._checkDirectory(`${this.tmpDir}/backup/files`, true);
1146-
this.removeTempBackupDir();
1155+
/**
1156+
* Validate an unpacked legacy backup in the temporary directory
1157+
*/
1158+
private _validateLegacyTempDir(): void {
1159+
console.log(`host.${this.hostname} Starting validation ...`);
1160+
let backupJSON;
1161+
try {
1162+
backupJSON = fs.readJSONSync(`${this.tmpDir}/backup/backup.json`);
1163+
} catch (e) {
1164+
throw new Error(`Backup corrupted. Backup does not contain a valid backup.json file: ${e.message}`);
1165+
}
11471166

1148-
resolve();
1149-
} catch (err) {
1150-
console.error(`host.${this.hostname} Backup corrupted: ${err.message}`);
1151-
return void this.processExit(EXIT_CODES.CANNOT_EXTRACT_FROM_ZIP);
1152-
}
1153-
},
1154-
);
1155-
});
1167+
if (!backupJSON || !backupJSON.objects || !backupJSON.objects.length) {
1168+
throw new Error(`host.${this.hostname} Backup corrupted. Backup does not contain valid objects`);
1169+
}
1170+
1171+
console.log(`host.${this.hostname} backup.json OK`);
1172+
1173+
try {
1174+
this._checkDirectory(`${this.tmpDir}/backup/files`, true);
1175+
} catch (e) {
1176+
throw new Error(`Backup corrupted: ${e.message}`);
1177+
}
11561178
}
11571179

11581180
/**
@@ -1204,7 +1226,7 @@ export class BackupRestore {
12041226
backups.sort((a, b) => (b > a ? 1 : b === a ? 0 : -1));
12051227
if (backups.length) {
12061228
backups.forEach((backup, i) =>
1207-
console.log(`${backup} or ${backup.replace(`_backup${tools.appName}.tar.gz`, '')} or ${i}`),
1229+
console.log(`${backup} or ${backup.replace(`${this.BACKUP_POSTFIX}.tar.gz`, '')} or ${i}`),
12081230
);
12091231
} else {
12101232
console.warn('No backups found');
@@ -1229,7 +1251,7 @@ export class BackupRestore {
12291251
if (backups.length) {
12301252
console.log('Please specify one of the backup names:');
12311253
backups.forEach((backup, i) =>
1232-
console.log(`${backup} or ${backup.replace(`_backup${tools.appName}.tar.gz`, '')} or ${i}`),
1254+
console.log(`${backup} or ${backup.replace(`${this.BACKUP_POSTFIX}.tar.gz`, '')} or ${i}`),
12331255
);
12341256
}
12351257
} else {
@@ -1240,9 +1262,9 @@ export class BackupRestore {
12401262
name = name.toString().replace(/\\/g, '/');
12411263
if (!name.includes('/')) {
12421264
name = BackupRestore.getBackupDir() + name;
1243-
const regEx = new RegExp(`_backup${tools.appName}`, 'i');
1265+
const regEx = new RegExp(this.BACKUP_POSTFIX, 'i');
12441266
if (!regEx.test(name)) {
1245-
name += `_backup${tools.appName}`;
1267+
name += this.BACKUP_POSTFIX;
12461268
}
12471269
if (!name.match(/\.tar\.gz$/i)) {
12481270
name += '.tar.gz';
@@ -1272,10 +1294,10 @@ export class BackupRestore {
12721294

12731295
if (
12741296
!(await fs.pathExists(path.join(backupBasePath, 'backup.json'))) &&
1275-
!(await fs.pathExists(path.join(backupBasePath, 'config.json')))
1297+
!(await fs.pathExists(path.join(backupBasePath, 'objects.jsonl')))
12761298
) {
12771299
console.error(
1278-
`host.${this.hostname} Cannot find extracted file "${path.join(backupBasePath, 'backup.json')}" or "${path.join(backupBasePath, 'config.json')}"`,
1300+
`host.${this.hostname} Cannot find extracted file "${path.join(backupBasePath, 'backup.json')}" or "${path.join(backupBasePath, 'objects.jsonl')}"`,
12791301
);
12801302
return { exitCode: EXIT_CODES.CANNOT_EXTRACT_FROM_ZIP, objects: this.objects, states: this.states };
12811303
}

‎packages/controller/test/lib/testConsole.ts‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -581,10 +581,16 @@ export function register(it: Mocha.TestFunction, expect: Chai.ExpectStatic, cont
581581

582582
// expect(found).to.be.true;
583583

584-
const name = Math.round(Math.random() * 10000).toString();
584+
const name = Math.round(Math.random() * 10_000).toString();
585585
res = await execAsync(`"${process.execPath}" "${iobExecutable}" backup ${name}`);
586586
expect(res.stderr).to.be.not.ok;
587-
expect(fs.existsSync(`${BackupRestore.getBackupDir() + name}.tar.gz`)).to.be.true;
587+
expect(fs.existsSync(`${BackupRestore.getBackupDir() + name}_backupiobroker.tar.gz`)).to.be.true;
588+
}).timeout(20_000);
589+
590+
it(`${testName}validates backup`, async () => {
591+
const res = await execAsync(`"${process.execPath}" "${iobExecutable}" validate 0`);
592+
expect(res.stderr).to.be.not.ok;
593+
expect(res.stdout).to.include('Backup OK');
588594
}).timeout(20_000);
589595

590596
// list l

0 commit comments

Comments
 (0)