Merge pull request #169 from celonis/TA-2737-batch-import-command

[TA-2737] Batch import command

src/api/batch-import-export-api.ts

@@ -1,10 +1,11 @@
 import {
     PackageExportTransport,
-    PackageKeyAndVersionPair,
+    PackageKeyAndVersionPair, PostPackageImportData,
     VariableManifestTransport
 } from "../interfaces/package-export-transport";
 import {httpClientV2} from "../services/http-client-service.v2";
 import {FatalError} from "../util/logger";
+import * as FormData from "form-data";
 
 class BatchImportExportApi {
     public static readonly INSTANCE = new BatchImportExportApi();
@@ -41,6 +42,14 @@ class BatchImportExportApi {
         });
     }
 
+    public importPackages(data: FormData, overwrite: boolean): Promise<PostPackageImportData[]> {
+        return httpClientV2.postFile(
+            "/package-manager/api/core/packages/import/batch",
+            data,
+            {overwrite}
+        );
+    }
+
     public findVariablesWithValuesByPackageKeysAndVersion(packagesByKeyAndVersion: PackageKeyAndVersionPair[]): Promise<VariableManifestTransport[]> {
         return httpClientV2.post("/package-manager/api/core/packages/export/batch/variables-with-assignments", packagesByKeyAndVersion).catch(e => {
             throw new FatalError(`Problem exporting package variables: ${e}`);

src/commands/config.command.ts

@@ -22,4 +22,8 @@ export class ConfigCommand {
     public batchExportPackages(packageKeys: string[], withDependencies: boolean = false): Promise<void> {
         return batchImportExportService.batchExportPackages(packageKeys, withDependencies);
     }
+
+    public batchImportPackages(file: string, overwrite: boolean): Promise<void> {
+        return batchImportExportService.batchImportPackages(file, overwrite);
+    }
 }

src/content-cli-config.ts

@@ -34,7 +34,7 @@ export class Config {
             .action(async cmd => {
                 await new ConfigCommand().listVariables(cmd.json, cmd.keysByVersion, cmd.keysByVersionFile);
                 process.exit();
-            })
+            });
 
         return program;
     }
@@ -53,6 +53,21 @@ export class Config {
 
         return program;
     }
+
+    public static import(program: CommanderStatic): CommanderStatic {
+        program
+            .command("import")
+            .description("Command to import package configs")
+            .option("-p, --profile <profile>", "Profile which you want to use to import packages")
+            .option("--overwrite", "Flag to allow overwriting of packages")
+            .requiredOption("-f, --file <file>", "Exported packages file (relative path)")
+            .action(async cmd => {
+                await new ConfigCommand().batchImportPackages(cmd.file, cmd.overwrite);
+                process.exit();
+            });
+
+        return program;
+    }
 }
 
 process.on("unhandledRejection", (e, promise) => {
@@ -63,6 +78,7 @@ const loadAllCommands = () => {
     Config.list(commander);
     Config.listVariables(commander);
     Config.export(commander);
+    Config.import(commander);
     commander.parse(process.argv);
 };
 

src/interfaces/batch-export-import-constants.ts

@@ -0,0 +1,6 @@
+export enum BatchExportImportConstants {
+    STUDIO_FILE_NAME = "studio.yml",
+    VARIABLES_FILE_NAME = "variables.yml",
+    MANIFEST_FILE_NAME = "manifest.yml",
+    STUDIO = "STUDIO"
+}

src/interfaces/package-export-transport.ts

@@ -24,8 +24,6 @@ export interface PackageManifestTransport {
     packageKey: string;
     flavor: string;
     activeVersion: string;
-    space?: SpaceTransport;
-    variableAssignments?: VariablesAssignments[];
     dependenciesByVersion: Map<string, DependencyTransport[]>;
 }
 
@@ -72,4 +70,14 @@ export interface StudioPackageManifest {
     packageKey: string;
     space: Partial<SpaceTransport>;
     runtimeVariableAssignments: VariablesAssignments[];
+}
+
+export interface PackageVersionImport {
+    oldVersion: string;
+    newVersion: string;
+}
+
+export interface PostPackageImportData {
+    packageKey: string;
+    importedVersions: PackageVersionImport[];
 }

src/services/http-client-service.v2.ts

@@ -45,10 +45,8 @@ class HttpClientServiceV2 {
         });
     }
 
-    public async postFile(url: string, body: any, parameters?: {}): Promise<any> {
+    public async postFile(url: string, formData: FormData, parameters?: {}): Promise<any> {
         return new Promise<any>((resolve, reject) => {
-            const formData = new FormData();
-            formData.append("package", body.formData.package);
             axios.post(
                 this.resolveUrl(url),
                 formData,

src/services/package-manager/batch-import-export-service.ts

@@ -11,6 +11,9 @@ import {FileService, fileService} from "../file-service";
 import {studioService} from "../studio/studio.service";
 import {parse, stringify} from "../../util/yaml"
 import AdmZip = require("adm-zip");
+import * as fs from "fs";
+import * as FormData from "form-data";
+import {BatchExportImportConstants} from "../../interfaces/batch-export-import-constants";
 
 class BatchImportExportService {
 
@@ -40,20 +43,20 @@ class BatchImportExportService {
         const exportedPackagesZip: AdmZip = new AdmZip(exportedPackagesData);
 
         const manifest: PackageManifestTransport[] = parse(
-            exportedPackagesZip.getEntry("manifest.yml").getData().toString()
+            exportedPackagesZip.getEntry(BatchExportImportConstants.MANIFEST_FILE_NAME).getData().toString()
         );
 
         const versionsByPackageKey = this.getVersionsByPackageKey(manifest);
 
         let exportedVariables = await this.getVersionedVariablesForPackagesWithKeys(versionsByPackageKey);
         exportedVariables = studioService.fixConnectionVariables(exportedVariables);
-        exportedPackagesZip.addFile("variables.yml", Buffer.from(stringify(exportedVariables), "utf8"));
+        exportedPackagesZip.addFile(BatchExportImportConstants.VARIABLES_FILE_NAME, Buffer.from(stringify(exportedVariables), "utf8"));
 
-        const studioPackageKeys = manifest.filter(packageManifest => packageManifest.flavor === "STUDIO")
+        const studioPackageKeys = manifest.filter(packageManifest => packageManifest.flavor === BatchExportImportConstants.STUDIO)
             .map(packageManifest => packageManifest.packageKey);
 
         const studioData = await studioService.getStudioPackageManifests(studioPackageKeys);
-        exportedPackagesZip.addFile("studio.yml", Buffer.from(stringify(studioData), "utf8"));
+        exportedPackagesZip.addFile(BatchExportImportConstants.STUDIO_FILE_NAME, Buffer.from(stringify(studioData), "utf8"));
 
         exportedPackagesZip.getEntries().forEach(entry => {
             if (entry.name.endsWith(".zip") && studioPackageKeys.includes(entry.name.split("_")[0])) {
@@ -69,6 +72,19 @@ class BatchImportExportService {
         logger.info(fileDownloadedMessage + filename);
     }
 
+    public async batchImportPackages(file: string, overwrite: boolean): Promise<void> {
+        const configs = new AdmZip(file);
+
+        const formData = this.buildBodyForImport(file, configs);
+
+        const postPackageImportData = await batchImportExportApi.importPackages(formData, overwrite);
+        await studioService.processImportedPackages(configs);
+
+        const reportFileName = "config_import_report_" + uuidv4() + ".json";
+        fileService.writeToFileWithGivenName(JSON.stringify(postPackageImportData), reportFileName);
+        logger.info("Config import report file: " + reportFileName);
+    }
+
     private exportListOfPackages(packages: PackageExportTransport[]): void {
         const filename = uuidv4() + ".json";
         fileService.writeToFileWithGivenName(JSON.stringify(packages), filename);
@@ -97,6 +113,21 @@ class BatchImportExportService {
 
         return batchImportExportApi.findVariablesWithValuesByPackageKeysAndVersion(variableExportRequest)
     }
+
+    private buildBodyForImport(file: string, configs: AdmZip): FormData {
+        const formData = new FormData();
+
+        formData.append("file", fs.createReadStream(file));
+
+        const variablesEntry = configs.getEntry(BatchExportImportConstants.VARIABLES_FILE_NAME);
+        if (variablesEntry) {
+            formData.append("mappedVariables", JSON.stringify(parse(variablesEntry.getData().toString())), {
+                contentType: "application/json"
+            });
+        }
+
+        return formData;
+    }
 }
 
 export const batchImportExportService = new BatchImportExportService();

src/services/package-manager/package-service.ts

@@ -22,6 +22,7 @@ import {ManifestDependency, ManifestNodeTransport} from "../../interfaces/manife
 import {DataPoolInstallVersionReport} from "../../interfaces/data-pool-manager.interfaces";
 import {SemanticVersioning} from "../../util/semantic-versioning"
 import {stringify} from "../../util/yaml";
+import * as FormData from "form-data";
 
 class PackageService {
     protected readonly fileDownloadedMessage = "File downloaded successfully. New filename: ";
@@ -184,7 +185,7 @@ class PackageService {
         let nodeInTargetTeam = await nodeApi.findOneByKeyAndRootNodeKey(packageToImport.packageKey, packageToImport.packageKey);
 
         const pathToZipFile = path.resolve(importedFilePath, packageToImport.packageKey + "_" + versionOfPackageBeingImported + ".zip");
-        const packageZip = await this.createBodyForImport(pathToZipFile);
+        const packageZip = this.createBodyForImport(pathToZipFile);
 
         await packageApi.importPackage(packageZip, targetSpace.id, !!nodeInTargetTeam, excludeActionFlows);
 
@@ -270,12 +271,11 @@ class PackageService {
         return importedPackages.includes(version);
     }
 
-    private async createBodyForImport(filename: string): Promise<object> {
-        return {
-            formData: {
-                package: await fs.createReadStream(filename, {encoding: null})
-            },
-        }
+    private createBodyForImport(filename: string): FormData {
+        const formData = new FormData();
+        formData.append("package", fs.createReadStream(filename, {encoding: null}));
+
+        return formData;
     }
 
     private async getTargetSpaceForExportedPackage(packageToImport: ManifestNodeTransport, spaceMappings: Map<string, string>): Promise<SpaceTransport> {

src/services/studio/studio.service.ts

@@ -20,6 +20,9 @@ import {nodeApi} from "../../api/node-api";
 import {variablesApi} from "../../api/variables-api";
 import {spaceApi} from "../../api/space-api";
 import {SpaceTransport} from "../../interfaces/save-space.interface";
+import {spaceService} from "../package-manager/space-service";
+import {variableService} from "../package-manager/variable-service";
+import {BatchExportImportConstants} from "../../interfaces/batch-export-import-constants";
 
 class StudioService {
 
@@ -78,6 +81,19 @@ class StudioService {
         return packageZip;
     }
 
+    public async processImportedPackages(configs: AdmZip): Promise<void> {
+        const studioFile = configs.getEntry(BatchExportImportConstants.STUDIO_FILE_NAME);
+
+        if (studioFile) {
+            const studioManifests: StudioPackageManifest[] = parse(configs.getEntry(BatchExportImportConstants.STUDIO_FILE_NAME).getData().toString());
+
+            await Promise.all(studioManifests.map(async manifest => {
+                await this.movePackageToSpace(manifest);
+                await this.assignRuntimeVariables(manifest);
+            }));
+        }
+    }
+
     private setSpaceIdForStudioPackages(packages: PackageExportTransport[], studioPackages: PackageWithVariableAssignments[]): PackageExportTransport[] {
         const studioPackageByKey = new Map<string, PackageWithVariableAssignments>();
         studioPackages.forEach(pkg => studioPackageByKey.set(pkg.key, pkg));
@@ -150,6 +166,25 @@ class StudioService {
 
         return variablesByKey;
     }
+
+    private async movePackageToSpace(manifest: StudioPackageManifest): Promise<void> {
+        const nodeInTargetTeam = await nodeApi.findOneByKeyAndRootNodeKey(manifest.packageKey, manifest.packageKey);
+
+        const allSpaces = await spaceService.refreshAndGetAllSpaces();
+        let targetSpace = allSpaces.find(space => space.name === manifest.space.name);
+
+        if (!targetSpace) {
+            targetSpace = await spaceService.createSpace(manifest.space.name, manifest.space.iconReference);
+        }
+
+        await packageApi.movePackageToSpace(nodeInTargetTeam.id, targetSpace.id);
+    }
+
+    private async assignRuntimeVariables(manifest: StudioPackageManifest): Promise<void> {
+        if (manifest.runtimeVariableAssignments.length) {
+            await variableService.assignVariableValues(manifest.packageKey, manifest.runtimeVariableAssignments);
+        }
+    }
 }
 
 export const studioService = new StudioService();

tests/config/config-export.spec.ts

@@ -19,6 +19,7 @@ import {
     VariableDefinition,
     VariablesAssignments
 } from "../../src/interfaces/package-manager.interfaces";
+import {BatchExportImportConstants} from "../../src/interfaces/batch-export-import-constants";
 
 describe("Config export", () => {
 
@@ -33,8 +34,8 @@ describe("Config export", () => {
 
     it("Should export studio file for studio packageKeys", async () => {
         const manifest: PackageManifestTransport[] = [];
-        manifest.push(ConfigUtils.buildManifestForKeyAndFlavor("key-1", "STUDIO"));
-        manifest.push(ConfigUtils.buildManifestForKeyAndFlavor("key-2", "STUDIO"));
+        manifest.push(ConfigUtils.buildManifestForKeyAndFlavor("key-1", BatchExportImportConstants.STUDIO));
+        manifest.push(ConfigUtils.buildManifestForKeyAndFlavor("key-2", BatchExportImportConstants.STUDIO));
         manifest.push(ConfigUtils.buildManifestForKeyAndFlavor("key-3", "TEST"));
         const exportedPackagesZip = ConfigUtils.buildBatchExportZip(manifest, []);
 
@@ -63,7 +64,7 @@ describe("Config export", () => {
         const fileBuffer = mockWriteSync.mock.calls[0][1];
         const actualZip = new AdmZip(fileBuffer);
 
-        const studioManifest: StudioPackageManifest[] = parse(actualZip.getEntry("studio.yml").getData().toString());
+        const studioManifest: StudioPackageManifest[] = parse(actualZip.getEntry(BatchExportImportConstants.STUDIO_FILE_NAME).getData().toString());
         expect(studioManifest).toHaveLength(2);
         expect(studioManifest).toContainEqual({
             packageKey: firstStudioPackage.key,
@@ -92,8 +93,8 @@ describe("Config export", () => {
         secondPackageDependencies.set("1.1.1", []);
 
         const manifest: PackageManifestTransport[] = [];
-        manifest.push(ConfigUtils.buildManifestForKeyAndFlavor("key-1", "STUDIO", firstPackageDependencies));
-        manifest.push(ConfigUtils.buildManifestForKeyAndFlavor("key-2", "STUDIO", secondPackageDependencies));
+        manifest.push(ConfigUtils.buildManifestForKeyAndFlavor("key-1", BatchExportImportConstants.STUDIO, firstPackageDependencies));
+        manifest.push(ConfigUtils.buildManifestForKeyAndFlavor("key-2", BatchExportImportConstants.STUDIO, secondPackageDependencies));
 
         const firstPackageVariableDefinition: VariableDefinition[] = [
             {
@@ -176,7 +177,7 @@ describe("Config export", () => {
         const fileBuffer = mockWriteSync.mock.calls[0][1];
         const actualZip = new AdmZip(fileBuffer);
 
-        const exportedVariablesFileContent: VariableManifestTransport[] = parse(actualZip.getEntry("variables.yml").getData().toString());
+        const exportedVariablesFileContent: VariableManifestTransport[] = parse(actualZip.getEntry(BatchExportImportConstants.VARIABLES_FILE_NAME).getData().toString());
         expect(exportedVariablesFileContent).toHaveLength(2);
         expect(exportedVariablesFileContent).toContainEqual({
             packageKey: "key-1",
@@ -233,8 +234,8 @@ describe("Config export", () => {
 
     it("Should remove SCENARIO asset files of packages", async () => {
         const manifest: PackageManifestTransport[] = [];
-        manifest.push(ConfigUtils.buildManifestForKeyAndFlavor("key-1", "STUDIO"));
-        manifest.push(ConfigUtils.buildManifestForKeyAndFlavor("key-2", "STUDIO"));
+        manifest.push(ConfigUtils.buildManifestForKeyAndFlavor("key-1", BatchExportImportConstants.STUDIO));
+        manifest.push(ConfigUtils.buildManifestForKeyAndFlavor("key-2", BatchExportImportConstants.STUDIO));
 
         const firstPackageNode = ConfigUtils.buildPackageNode("key-1", "");
         const firstPackageScenarioChild = ConfigUtils.buildChildNode("child-1-scenario", firstPackageNode.key, "SCENARIO");
@@ -275,8 +276,8 @@ describe("Config export", () => {
 
     it("Should add appName to metadata for CONNECTION variables of package.yml files", async () => {
         const manifest: PackageManifestTransport[] = [];
-        manifest.push(ConfigUtils.buildManifestForKeyAndFlavor("key-1", "STUDIO"));
-        manifest.push(ConfigUtils.buildManifestForKeyAndFlavor("key-2", "STUDIO"));
+        manifest.push(ConfigUtils.buildManifestForKeyAndFlavor("key-1", BatchExportImportConstants.STUDIO));
+        manifest.push(ConfigUtils.buildManifestForKeyAndFlavor("key-2", BatchExportImportConstants.STUDIO));
 
         const firstPackageVariableDefinition: VariableDefinition[] = [
             {