Merge pull request #167 from celonis/TA-2736-batch-export-command

[TA-2736] Batch export command

src/api/batch-import-export-api.ts

@@ -1,4 +1,8 @@
-import {PackageExportTransport} from "../interfaces/package-export-transport";
+import {
+    PackageExportTransport,
+    PackageKeyAndVersionPair,
+    VariableManifestTransport
+} from "../interfaces/package-export-transport";
 import {httpClientV2} from "../services/http-client-service.v2";
 import {FatalError} from "../util/logger";
 
@@ -26,6 +30,22 @@ class BatchImportExportApi {
             throw new FatalError(`Problem getting active packages by keys: ${e}`);
         });
     }
+
+    public exportPackages(packageKeys: string[], withDependencies: boolean = false): Promise<Buffer> {
+        const queryParams = new URLSearchParams();
+        packageKeys.forEach(packageKey => queryParams.append("packageKeys", packageKey));
+        queryParams.set("withDependencies", withDependencies.toString());
+
+        return httpClientV2.getFile(`/package-manager/api/core/packages/export/batch?${queryParams.toString()}`).catch(e => {
+            throw new FatalError(`Problem exporting packages: ${e}`);
+        });
+    }
+
+    public findVariablesWithValuesByPackageKeysAndVersion(packagesByKeyAndVersion: PackageKeyAndVersionPair[]): Promise<VariableManifestTransport[]> {
+        return httpClientV2.post("/package-manager/api/core/packages/export/batch/variables-with-assignments", packagesByKeyAndVersion).catch(e => {
+            throw new FatalError(`Problem exporting package variables: ${e}`);
+        })
+    }
 }
 
 export const batchImportExportApi = BatchImportExportApi.INSTANCE;

src/api/variables-api.ts

@@ -29,6 +29,12 @@ class VariablesApi {
             throw new FatalError(`Problem getting variables assignment values for type ${type}: ${e}`);
         });
     }
+
+    public getRuntimeVariableValues(packageKey: string): Promise<VariablesAssignments[]> {
+        return httpClientV2.get(`/package-manager/api/nodes/by-package-key/${packageKey}/variables/runtime-values`).catch(e => {
+            throw new FatalError(`Problem getting runtime variables of package ${packageKey}: ${e}`);
+        });
+    }
 }
 
 export const variablesApi = VariablesApi.INSTANCE;

src/commands/config.command.ts

@@ -9,4 +9,8 @@ export class ConfigCommand {
             await batchImportExportService.listActivePackages(flavors ?? []);
         }
     }
+
+    public batchExportPackages(packageKeys: string[], withDependencies: boolean = false): Promise<void> {
+        return batchImportExportService.batchExportPackages(packageKeys, withDependencies);
+    }
 }

src/content-cli-config.ts

@@ -22,6 +22,21 @@ export class Config {
 
         return program;
     }
+
+    public static export(program: CommanderStatic): CommanderStatic {
+        program
+            .command("export")
+            .description("Command to export package configs")
+            .option("-p, --profile <profile>", "Profile which you want to use to list packages")
+            .requiredOption("--packageKeys <packageKeys...>", "Keys of packages to export")
+            .option("--withDependencies", "Include variables and dependencies", "")
+            .action(async cmd => {
+                await new ConfigCommand().batchExportPackages(cmd.packageKeys, cmd.withDependencies);
+                process.exit();
+            });
+
+        return program;
+    }
 }
 
 process.on("unhandledRejection", (e, promise) => {
@@ -30,6 +45,7 @@ process.on("unhandledRejection", (e, promise) => {
 
 const loadAllCommands = () => {
     Config.list(commander);
+    Config.export(commander);
     commander.parse(process.argv);
 };
 

src/interfaces/package-export-transport.ts

@@ -1,4 +1,5 @@
-import {StudioComputeNodeDescriptor} from "./package-manager.interfaces";
+import {StudioComputeNodeDescriptor, VariableDefinition, VariablesAssignments} from "./package-manager.interfaces";
+import {SpaceTransport} from "./save-space.interface";
 
 export interface DependencyTransport {
     key: string;
@@ -17,4 +18,58 @@ export interface PackageExportTransport {
     dependencies: DependencyTransport[];
     spaceId?: string;
     datamodels?: StudioComputeNodeDescriptor[];
+}
+
+export interface PackageManifestTransport {
+    packageKey: string;
+    flavor: string;
+    activeVersion: string;
+    space?: SpaceTransport;
+    variableAssignments?: VariablesAssignments[];
+    dependenciesByVersion: Map<string, DependencyTransport[]>;
+}
+
+export interface VariableExportTransport {
+    key: string;
+    value: object;
+    type: string;
+    metadata: object;
+}
+
+export interface VariableManifestTransport {
+    packageKey: string;
+    version: string;
+    variables?: VariableExportTransport[];
+}
+
+export interface PackageKeyAndVersionPair {
+    packageKey: string;
+    version: string;
+}
+
+export interface NodeExportTransport {
+    key: string;
+    parentNodeKey: string;
+    packageNodeKey: string;
+    name: string;
+    type: string;
+    exportSerializationType: string;
+    serializedContent: string;
+    schemaVersion: number;
+
+    unversionedMetadata: object;
+    versionedMetdata: object;
+
+    invalidContent?: boolean;
+    serializedDocument: Buffer;
+}
+
+export interface NodeSerializedContent {
+    variables: VariableDefinition[]
+}
+
+export interface StudioPackageManifest {
+    packageKey: string;
+    space: Partial<SpaceTransport>;
+    runtimeVariableAssignments: VariablesAssignments[];
 }

src/interfaces/package-manager.interfaces.ts

@@ -58,7 +58,7 @@ export interface VariablesAssignments {
     type: string;
 }
 
-export interface VariableDefinitionWithValue {
+export interface VariableDefinition {
     key: string;
     type: PackageManagerVariableType;
     description?: string;

src/services/http-client-service.v2.ts

@@ -22,6 +22,29 @@ class HttpClientServiceV2 {
         })
     }
 
+    public async getFile(url: string): Promise<any> {
+        return new Promise<any>((resolve, reject) => {
+            axios.get(this.resolveUrl(url), {
+                headers: this.buildHeaders(contextService.getContext().profile),
+                responseType: "stream"
+            }).then(response => {
+                const data: Buffer[] = [];
+                response.data.on("data", (chunk: Buffer) => {
+                    data.push(chunk);
+                });
+                response.data.on("end", () => {
+                    if (this.checkBadRequest(response.status)) {
+                        this.handleBadRequest(response.status, response.data, reject);
+                    } else {
+                        this.handleResponseStreamData(Buffer.concat(data), resolve, reject);
+                    }
+                });
+            }).catch(err => {
+                this.handleError(err, resolve, reject);
+            })
+        });
+    }
+
     public async postFile(url: string, body: any, parameters?: {}): Promise<any> {
         return new Promise<any>((resolve, reject) => {
             const formData = new FormData();

src/services/package-manager/batch-import-export-service.ts

@@ -1,9 +1,16 @@
 import {batchImportExportApi} from "../../api/batch-import-export-api";
 import {logger} from "../../util/logger";
 import {v4 as uuidv4} from "uuid";
-import {PackageExportTransport} from "../../interfaces/package-export-transport";
+import {
+    PackageExportTransport,
+    PackageKeyAndVersionPair,
+    PackageManifestTransport,
+    VariableManifestTransport
+} from "../../interfaces/package-export-transport";
 import {FileService, fileService} from "../file-service";
 import {studioService} from "../studio/studio.service";
+import {parse, stringify} from "../../util/yaml"
+import AdmZip = require("adm-zip");
 
 class BatchImportExportService {
 
@@ -28,11 +35,68 @@ class BatchImportExportService {
         this.exportListOfPackages(packagesToExport);
     }
 
+    public async batchExportPackages(packageKeys: string[], withDependencies: boolean = false): Promise<void> {
+        const exportedPackagesData: Buffer = await batchImportExportApi.exportPackages(packageKeys, withDependencies);
+        const exportedPackagesZip: AdmZip = new AdmZip(exportedPackagesData);
+
+        const manifest: PackageManifestTransport[] = parse(
+            exportedPackagesZip.getEntry("manifest.yml").getData().toString()
+        );
+
+        const versionsByPackageKey = this.getVersionsByPackageKey(manifest);
+
+        let exportedVariables = await this.getVersionedVariablesForPackagesWithKeys(versionsByPackageKey);
+        exportedVariables = studioService.fixConnectionVariables(exportedVariables);
+        exportedPackagesZip.addFile("variables.yml", Buffer.from(stringify(exportedVariables), "utf8"));
+
+        const studioPackageKeys = manifest.filter(packageManifest => packageManifest.flavor === "STUDIO")
+            .map(packageManifest => packageManifest.packageKey);
+
+        const studioData = await studioService.getStudioPackageManifests(studioPackageKeys);
+        exportedPackagesZip.addFile("studio.yml", Buffer.from(stringify(studioData), "utf8"));
+
+        exportedPackagesZip.getEntries().forEach(entry => {
+            if (entry.name.endsWith(".zip") && studioPackageKeys.includes(entry.name.split("_")[0])) {
+                const updatedPackage = studioService.processPackageForExport(entry, exportedVariables);
+
+                exportedPackagesZip.updateFile(entry, updatedPackage.toBuffer());
+            }
+        });
+
+        const fileDownloadedMessage = "File downloaded successfully. New filename: ";
+        const filename = `export_${uuidv4()}.zip`;
+        exportedPackagesZip.writeZip(filename);
+        logger.info(fileDownloadedMessage + filename);
+    }
+
     private exportListOfPackages(packages: PackageExportTransport[]): void {
         const filename = uuidv4() + ".json";
         fileService.writeToFileWithGivenName(JSON.stringify(packages), filename);
         logger.info(FileService.fileDownloadedMessage + filename);
     }
+
+    private getVersionsByPackageKey(manifests: PackageManifestTransport[]): Map<string, string[]> {
+        const versionsByPackageKey = new Map<string, string[]>();
+        manifests.forEach(packageManifest => {
+            versionsByPackageKey.set(packageManifest.packageKey, Object.keys(packageManifest.dependenciesByVersion));
+        })
+
+        return versionsByPackageKey;
+    }
+
+    private getVersionedVariablesForPackagesWithKeys(versionsByPackageKey: Map<string, string[]>): Promise<VariableManifestTransport[]> {
+        const variableExportRequest: PackageKeyAndVersionPair[] = [];
+        versionsByPackageKey?.forEach((versions, key) => {
+            versions?.forEach(version => {
+                variableExportRequest.push({
+                    packageKey: key,
+                    version: version,
+                })
+            })
+        });
+
+        return batchImportExportApi.findVariablesWithValuesByPackageKeysAndVersion(variableExportRequest)
+    }
 }
 
 export const batchImportExportService = new BatchImportExportService();