Skip to content

Commit

Permalink
chore: Format appsmithctl code (#37532)
Browse files Browse the repository at this point in the history 
Formatting just before merging `appsmithctl` code into RTS.

## Automation

/test sanity

### 🔍 Cypress test results
<!-- This is an auto-generated comment: Cypress test results  -->
> [!TIP]
> 🟢 🟢 🟢 All cypress tests have passed! 🎉 🎉 🎉
> Workflow run:
<https://github.com/appsmithorg/appsmith/actions/runs/11890257914>
> Commit: 5c83801
> <a
href="https://internal.appsmith.com/app/cypress-dashboard/rundetails-65890b3c81d7400d08fa9ee5?branch=master&workflowId=11890257914&attempt=1"
target="_blank">Cypress dashboard</a>.
> Tags: `@tag.Sanity`
> Spec:
> <hr>Mon, 18 Nov 2024 10:46:20 UTC
<!-- end of auto-generated comment: Cypress test results  -->


## Communication
Should the DevRel and Marketing teams inform users about this change?
- [ ] Yes
- [x] No


<!-- This is an auto-generated comment: release notes by coderabbit.ai
-->
## Summary by CodeRabbit

## Release Notes

- **New Features**
- Added constants for backup and restore paths, database dump file
names, and email notifications related to backup errors.
	- Introduced a new function for running restore operations.

- **Bug Fixes**
	- Enhanced error handling and logging for backup and restore processes.

- **Documentation**
	- Improved clarity in console log messages across various utilities.

- **Refactor**
- Standardized string formatting and code structure for better
readability and maintainability across multiple files, including
consistent use of double quotes and improved error handling formatting.

- **Tests**
- Expanded test coverage for backup utilities and improved formatting
for better readability in test cases.
<!-- end of auto-generated comment: release notes by coderabbit.ai -->
  • Loading branch information
@sharat87
sharat87 authored Nov 19, 2024
1 parent 94a3add commit afd2fcc
Show file tree
Hide file tree
Showing 14 changed files with 933 additions and 605 deletions.
262 changes: 171 additions & 91 deletions deploy/docker/fs/opt/appsmith/utils/bin/backup.js
View file
Original file line number Diff line number Diff line change
@@ -1,12 +1,12 @@
const fsPromises = require('fs/promises');
const path = require('path');
const os = require('os');
const utils = require('./utils');
const Constants = require('./constants');
const logger = require('./logger');
const mailer = require('./mailer');
const tty = require('tty');
const readlineSync = require('readline-sync');
const fsPromises = require("fs/promises");
const path = require("path");
const os = require("os");
const utils = require("./utils");
const Constants = require("./constants");
const logger = require("./logger");
const mailer = require("./mailer");
const tty = require("tty");
const readlineSync = require("readline-sync");

const command_args = process.argv.slice(3);

Expand All @@ -19,9 +19,10 @@ async function run() {
await utils.ensureSupervisorIsRunning();

try {
console.log('Available free space at /appsmith-stacks');
const availSpaceInBytes = getAvailableBackupSpaceInBytes("/appsmith-stacks");
console.log('\n');
console.log("Available free space at /appsmith-stacks");
const availSpaceInBytes =
getAvailableBackupSpaceInBytes("/appsmith-stacks");
console.log("\n");

checkAvailableBackupSpace(availSpaceInBytes);

Expand All @@ -36,44 +37,67 @@ async function run() {

await createManifestFile(backupContentsPath);

if (!command_args.includes('--non-interactive') && (tty.isatty(process.stdout.fd))){
if (
!command_args.includes("--non-interactive") &&
tty.isatty(process.stdout.fd)
) {
encryptionPassword = getEncryptionPasswordFromUser();
if (encryptionPassword == -1){
throw new Error('Backup process aborted because a valid enctyption password could not be obtained from the user');
if (encryptionPassword == -1) {
throw new Error(
"Backup process aborted because a valid enctyption password could not be obtained from the user",
);
}
encryptArchive = true;
}
await exportDockerEnvFile(backupContentsPath, encryptArchive);

archivePath = await createFinalArchive(backupRootPath, timestamp);
// shell.exec("openssl enc -aes-256-cbc -pbkdf2 -iter 100000 -in " + archivePath + " -out " + archivePath + ".enc");
if (encryptArchive){
const encryptedArchivePath = await encryptBackupArchive(archivePath,encryptionPassword);
await logger.backup_info('Finished creating an encrypted a backup archive at ' + encryptedArchivePath);
if (archivePath != null) {
await fsPromises.rm(archivePath, { recursive: true, force: true });
}
}
else {
await logger.backup_info('Finished creating a backup archive at ' + archivePath);
console.log('********************************************************* IMPORTANT!!! *************************************************************');
console.log('*** Please ensure you have saved the APPSMITH_ENCRYPTION_SALT and APPSMITH_ENCRYPTION_PASSWORD variables from the docker.env file **')
console.log('*** These values are not included in the backup export. **');
console.log('************************************************************************************************************************************');
if (encryptArchive) {
const encryptedArchivePath = await encryptBackupArchive(
archivePath,
encryptionPassword,
);
await logger.backup_info(
"Finished creating an encrypted a backup archive at " +
encryptedArchivePath,
);
if (archivePath != null) {
await fsPromises.rm(archivePath, { recursive: true, force: true });
}
} else {
await logger.backup_info(
"Finished creating a backup archive at " + archivePath,
);
console.log(
"********************************************************* IMPORTANT!!! *************************************************************",
);
console.log(
"*** Please ensure you have saved the APPSMITH_ENCRYPTION_SALT and APPSMITH_ENCRYPTION_PASSWORD variables from the docker.env file **",
);
console.log(
"*** These values are not included in the backup export. **",
);
console.log(
"************************************************************************************************************************************",
);
}

await fsPromises.rm(backupRootPath, { recursive: true, force: true });

await logger.backup_info('Finished taking a backup at ' + archivePath);

await logger.backup_info("Finished taking a backup at " + archivePath);
} catch (err) {
errorCode = 1;
await logger.backup_error(err.stack);

if (command_args.includes('--error-mail')) {
if (command_args.includes("--error-mail")) {
const currentTS = new Date().getTime();
const lastMailTS = await utils.getLastBackupErrorMailSentInMilliSec();
if ((lastMailTS + Constants.DURATION_BETWEEN_BACKUP_ERROR_MAILS_IN_MILLI_SEC) < currentTS) {
if (
lastMailTS +
Constants.DURATION_BETWEEN_BACKUP_ERROR_MAILS_IN_MILLI_SEC <
currentTS
) {
await mailer.sendBackupErrorToAdmins(err, timestamp);
await utils.updateLastBackupErrorMailSentInMilliSec(currentTS);
}
Expand All @@ -92,137 +116,193 @@ async function run() {
}
}

async function encryptBackupArchive(archivePath, encryptionPassword){
const encryptedArchivePath = archivePath + '.enc';
await utils.execCommand(['openssl', 'enc', '-aes-256-cbc', '-pbkdf2', '-iter', 100000, '-in', archivePath, '-out', encryptedArchivePath, '-k', encryptionPassword ])
async function encryptBackupArchive(archivePath, encryptionPassword) {
const encryptedArchivePath = archivePath + ".enc";
await utils.execCommand([
"openssl",
"enc",
"-aes-256-cbc",
"-pbkdf2",
"-iter",
100000,
"-in",
archivePath,
"-out",
encryptedArchivePath,
"-k",
encryptionPassword,
]);
return encryptedArchivePath;
}

function getEncryptionPasswordFromUser(){
for (const _ of [1, 2, 3])
{
const encryptionPwd1 = readlineSync.question('Enter a password to encrypt the backup archive: ', { hideEchoBack: true });
const encryptionPwd2 = readlineSync.question('Enter the above password again: ', { hideEchoBack: true });
if (encryptionPwd1 === encryptionPwd2){
if (encryptionPwd1){
function getEncryptionPasswordFromUser() {
for (const _ of [1, 2, 3]) {
const encryptionPwd1 = readlineSync.question(
"Enter a password to encrypt the backup archive: ",
{ hideEchoBack: true },
);
const encryptionPwd2 = readlineSync.question(
"Enter the above password again: ",
{ hideEchoBack: true },
);
if (encryptionPwd1 === encryptionPwd2) {
if (encryptionPwd1) {
return encryptionPwd1;
}
console.error("Invalid input. Empty password is not allowed, please try again.")
}
else {
}
console.error(
"Invalid input. Empty password is not allowed, please try again.",
);
} else {
console.error("The passwords do not match, please try again.");
}
}
console.error("Aborting backup process, failed to obtain valid encryption password.");
return -1
console.error(
"Aborting backup process, failed to obtain valid encryption password.",
);
return -1;
}

async function exportDatabase(destFolder) {
console.log('Exporting database');
await executeMongoDumpCMD(destFolder, utils.getDburl())
console.log('Exporting database done.');
console.log("Exporting database");
await executeMongoDumpCMD(destFolder, utils.getDburl());
console.log("Exporting database done.");
}

async function createGitStorageArchive(destFolder) {
console.log('Creating git-storage archive');
console.log("Creating git-storage archive");

const gitRoot = getGitRoot(process.env.APPSMITH_GIT_ROOT);

await executeCopyCMD(gitRoot, destFolder)
await executeCopyCMD(gitRoot, destFolder);

console.log('Created git-storage archive');
console.log("Created git-storage archive");
}

async function createManifestFile(path) {
const version = await utils.getCurrentAppsmithVersion()
const manifest_data = { "appsmithVersion": version, "dbName": utils.getDatabaseNameFromMongoURI(utils.getDburl()) }
await fsPromises.writeFile(path + '/manifest.json', JSON.stringify(manifest_data));
const version = await utils.getCurrentAppsmithVersion();
const manifest_data = {
appsmithVersion: version,
dbName: utils.getDatabaseNameFromMongoURI(utils.getDburl()),
};
await fsPromises.writeFile(
path + "/manifest.json",
JSON.stringify(manifest_data),
);
}

async function exportDockerEnvFile(destFolder, encryptArchive) {
console.log('Exporting docker environment file');
const content = await fsPromises.readFile('/appsmith-stacks/configuration/docker.env', { encoding: 'utf8' });
console.log("Exporting docker environment file");
const content = await fsPromises.readFile(
"/appsmith-stacks/configuration/docker.env",
{ encoding: "utf8" },
);
let cleaned_content = removeSensitiveEnvData(content);
if (encryptArchive){
cleaned_content += '\nAPPSMITH_ENCRYPTION_SALT=' + process.env.APPSMITH_ENCRYPTION_SALT +
'\nAPPSMITH_ENCRYPTION_PASSWORD=' + process.env.APPSMITH_ENCRYPTION_PASSWORD
if (encryptArchive) {
cleaned_content +=
"\nAPPSMITH_ENCRYPTION_SALT=" +
process.env.APPSMITH_ENCRYPTION_SALT +
"\nAPPSMITH_ENCRYPTION_PASSWORD=" +
process.env.APPSMITH_ENCRYPTION_PASSWORD;
}
await fsPromises.writeFile(destFolder + '/docker.env', cleaned_content);
console.log('Exporting docker environment file done.');
await fsPromises.writeFile(destFolder + "/docker.env", cleaned_content);
console.log("Exporting docker environment file done.");
}

async function executeMongoDumpCMD(destFolder, appsmithMongoURI) {
return await utils.execCommand(['mongodump', `--uri=${appsmithMongoURI}`, `--archive=${destFolder}/mongodb-data.gz`, '--gzip']);// generate cmd
return await utils.execCommand([
"mongodump",
`--uri=${appsmithMongoURI}`,
`--archive=${destFolder}/mongodb-data.gz`,
"--gzip",
]); // generate cmd
}

async function createFinalArchive(destFolder, timestamp) {
console.log('Creating final archive');
console.log("Creating final archive");

const archive = `${Constants.BACKUP_PATH}/appsmith-backup-${timestamp}.tar.gz`;
await utils.execCommand(['tar', '-cah', '-C', destFolder, '-f', archive, '.']);

console.log('Created final archive');
await utils.execCommand([
"tar",
"-cah",
"-C",
destFolder,
"-f",
archive,
".",
]);

console.log("Created final archive");

return archive;
}

async function postBackupCleanup() {
console.log('Starting the cleanup task after taking a backup.');
let backupArchivesLimit = getBackupArchiveLimit(process.env.APPSMITH_BACKUP_ARCHIVE_LIMIT);
console.log("Starting the cleanup task after taking a backup.");
let backupArchivesLimit = getBackupArchiveLimit(
process.env.APPSMITH_BACKUP_ARCHIVE_LIMIT,
);
const backupFiles = await utils.listLocalBackupFiles();
while (backupFiles.length > backupArchivesLimit) {
const fileName = backupFiles.shift();
await fsPromises.rm(Constants.BACKUP_PATH + '/' + fileName);
await fsPromises.rm(Constants.BACKUP_PATH + "/" + fileName);
}
console.log('Cleanup task completed.');

console.log("Cleanup task completed.");
}
async function executeCopyCMD(srcFolder, destFolder) {
return await utils.execCommand(['ln', '-s', srcFolder, destFolder + '/git-storage'])
return await utils.execCommand([
"ln",
"-s",
srcFolder,
destFolder + "/git-storage",
]);
}

function getGitRoot(gitRoot) {
if (gitRoot == null || gitRoot === '') {
gitRoot = '/appsmith-stacks/git-storage';
if (gitRoot == null || gitRoot === "") {
gitRoot = "/appsmith-stacks/git-storage";
}
return gitRoot
return gitRoot;
}

function generateBackupRootPath() {
return fsPromises.mkdtemp(path.join(os.tmpdir(), 'appsmithctl-backup-'));
return fsPromises.mkdtemp(path.join(os.tmpdir(), "appsmithctl-backup-"));
}

function getBackupContentsPath(backupRootPath, timestamp) {
return backupRootPath + '/appsmith-backup-' + timestamp;
return backupRootPath + "/appsmith-backup-" + timestamp;
}

function removeSensitiveEnvData(content) {
// Remove encryption and Mongodb data from docker.env
const output_lines = []
content.split(/\r?\n/).forEach(line => {
if (!line.startsWith("APPSMITH_ENCRYPTION") && !line.startsWith("APPSMITH_MONGODB") && !line.startsWith("APPSMITH_DB_URL=")) {
const output_lines = [];
content.split(/\r?\n/).forEach((line) => {
if (
!line.startsWith("APPSMITH_ENCRYPTION") &&
!line.startsWith("APPSMITH_MONGODB") &&
!line.startsWith("APPSMITH_DB_URL=")
) {
output_lines.push(line);
}
});
return output_lines.join('\n')
return output_lines.join("\n");
}

function getBackupArchiveLimit(backupArchivesLimit) {
if (!backupArchivesLimit)
backupArchivesLimit = Constants.APPSMITH_DEFAULT_BACKUP_ARCHIVE_LIMIT;
return backupArchivesLimit
return backupArchivesLimit;
}

async function removeOldBackups(backupFiles, backupArchivesLimit) {
while (backupFiles.length > backupArchivesLimit) {
const fileName = backupFiles.shift();
await fsPromises.rm(Constants.BACKUP_PATH + '/' + fileName);
await fsPromises.rm(Constants.BACKUP_PATH + "/" + fileName);
}
return backupFiles
return backupFiles;
}

function getTimeStampInISO() {
return new Date().toISOString().replace(/:/g, '-')
return new Date().toISOString().replace(/:/g, "-");
}

async function getAvailableBackupSpaceInBytes(path) {
Expand All @@ -232,12 +312,12 @@ async function getAvailableBackupSpaceInBytes(path) {

function checkAvailableBackupSpace(availSpaceInBytes) {
if (availSpaceInBytes < Constants.MIN_REQUIRED_DISK_SPACE_IN_BYTES) {
throw new Error("Not enough space available at /appsmith-stacks. Please ensure availability of at least 2GB to backup successfully.");
throw new Error(
"Not enough space available at /appsmith-stacks. Please ensure availability of at least 2GB to backup successfully.",
);
}
}



module.exports = {
run,
getTimeStampInISO,
Expand Down
Loading

0 comments on commit afd2fcc

Please sign in to comment.