Adds metadata command (#218)

- Generates module metadata from module.yaml and existing metadata

closes #206

Signed-off-by: Sean Sundberg <seansund@us.ibm.com>

package.json

@@ -112,6 +112,8 @@
   "devDependencies": {
     "@commitlint/cli": "^11.0.0",
     "@commitlint/config-conventional": "^11.0.0",
+    "@types/event-stream": "^4.0.0",
+    "@types/fs-extra": "^9.0.13",
     "@types/inquirer": "^7.3.1",
     "@types/jest": "^26.0.20",
     "@types/js-yaml": "^4.0.0",
@@ -152,15 +154,20 @@
   "dependencies": {
     "ajv-cli": "^5.0.0",
     "compare-versions": "^3.6.0",
+    "event-stream": "^4.0.1",
+    "fs-extra": "^10.1.0",
     "inquirer": "^7.3.3",
     "inquirer-checkbox-plus": "^1.0.2",
     "inquirer-checkbox-plus-prompt": "^1.0.1",
     "inquirer-prompt-suggest": "^0.1.0",
+    "js-optional": "^1.0.4",
     "js-yaml": "^4.0.0",
     "lodash.clonedeep": "^4.5.0",
+    "simple-git": "^3.14.1",
     "superagent": "^6.1.0",
     "typescript-ioc": "^3.2.2",
-    "yargs": "^17.2.1"
+    "yargs": "^17.2.1",
+    "z-schema": "^5.0.4"
   },
   "homepage": "https://github.com/cloud-native-toolkit/iascable.git#readme"
 }

src/commands/iascable-metadata.ts

@@ -0,0 +1,209 @@
+import {Arguments, Argv} from 'yargs';
+import {Container} from 'typescript-ioc';
+import {promises} from 'fs';
+import {join} from 'path';
+import {RemoteWithRefs, simpleGit} from 'simple-git';
+import {dump} from 'js-yaml';
+import {get} from 'superagent';
+
+import {CommandLineInput} from './inputs/command-line.input';
+import {IascableGenerateInput} from './inputs/iascable.input';
+import {Module} from '../models';
+import {ModuleMetadataApi, ModuleServiceCreateResult} from '../services/module-metadata-service';
+import {LoggerApi} from '../util/logger';
+
+export const command = 'metadata';
+export const desc = 'Generates the metadata for a given module';
+export const builder = (yargs: Argv<any>) => {
+  return yargs
+    .option('moduleUrl', {
+      description: 'The version number of the metadata release',
+      demandOption: false,
+    })
+    .option('moduleVersion', {
+      description: 'The version number of the metadata release',
+      demandOption: false,
+    })
+    .option('repoSlug', {
+      alias: 'r',
+      description: 'The repo slug of the module in the form {git org}/{repo name}',
+      demandOption: false,
+    })
+    .option('metadataFile', {
+      alias: 'm',
+      description: 'The file containing the input metadata',
+      demandOption: false,
+      default: 'module.yaml'
+    })
+    .option('publishBranch', {
+      alias: 'b',
+      description: 'The branch where the module metadata has been published',
+      demandOption: false,
+      default: 'gh-pages'
+    })
+    .option('outDir', {
+      alias: 'o',
+      description: 'The base directory where the command output will be written',
+      demandOption: false,
+      default: './output'
+    })
+    .option('flattenOutput', {
+      alias: 'f',
+      description: 'Flag indicating the output path should be flattened. If false the documentation will be placed in {outDir}/{module}/README.md.',
+      demandOption: false,
+      default: false,
+      type: 'boolean'
+    })
+    .option('debug', {
+      type: 'boolean',
+      describe: 'Flag to turn on more detailed output message',
+    })
+    .middleware(async argv => {
+      const repoSlug: string = await getRepoSlug(argv.moduleUrl)
+      const moduleVersion: string = await getModuleVersion(repoSlug, argv.moduleVersion)
+
+      return {
+        repoSlug,
+        moduleVersion
+      }
+    })
+    .middleware(argv => {
+      const moduleVersion: string = argv.moduleVersion
+
+      if (!moduleVersion) {
+        return {}
+      }
+
+      return {
+        moduleVersion: 'v' + moduleVersion.replace(new RegExp('^v'), '')
+      }
+    })
+    .check(argv => {
+      if (!argv.repoSlug) {
+        throw new Error('Unable to retrieve the repoSlug for provided git repo')
+      }
+
+      return true
+    })
+    .check(argv => {
+      if (!argv.moduleVersion) {
+        throw new Error('Unable to determine the version for the module')
+      }
+
+      return true
+    });
+};
+
+export const handler = async (argv: Arguments<IascableGenerateInput & CommandLineInput>) => {
+  process.env.LOG_LEVEL = argv.debug ? 'debug' : 'info';
+
+  const cmd: ModuleMetadataApi = Container.get(ModuleMetadataApi);
+
+  const version: string = argv.moduleVersion
+  const repoSlug: string = argv.repoSlug
+  const metadataFile: string = argv.metadataFile
+  const publishBranch: string = argv.publishBranch
+
+  console.log(`Generating metadata for module: ${repoSlug}#${version}`)
+  try {
+    const result: ModuleServiceCreateResult = await cmd.create({version, repoSlug, metadataFile, publishBranch})
+
+    await outputResult(argv.outDir, result.metadata, argv)
+  } catch (err) {
+    console.log('')
+    console.error(`Error: ${err.message}`)
+  }
+};
+
+const outputResult = async (outputDir: string, module: Module, options: {flattenOutput: boolean}) => {
+
+  const moduleName: string = module.name
+  const basePath = options.flattenOutput ? outputDir : join(outputDir, moduleName)
+
+  await promises.mkdir(basePath, {recursive: true})
+
+  const filename = join(basePath, 'index.yaml')
+
+  console.log(`  Writing module metadata for ${moduleName}: ${filename}`)
+
+  await promises.writeFile(filename, dump(module))
+}
+
+const getModuleVersion =  async (repoSlug: string, providedVersion: string): Promise<string> => {
+  if (providedVersion) {
+    return providedVersion
+  }
+
+  if (!repoSlug) {
+    return ''
+  }
+
+  const location = await getLatestReleaseUrl(`https://github.com/${repoSlug}/releases/latest`)
+
+  const versionRegExp = new RegExp('https?://.*/v?([0-9]+)[.]([0-9]+)[.]([0-9]+)')
+  const match = versionRegExp.exec(location)
+  if (match) {
+    const major = match[1]
+    const minor = match[2]
+    const patch = match[3]
+
+    return `${major}.${minor}.${parseInt(patch, 10) + 1}`
+  }
+
+  return ''
+}
+
+const getLatestReleaseUrl = async (url: string): Promise<string> => {
+  const location: string = await get(url)
+    .redirects(0)
+    .then(resp => {
+      return resp.headers['location']
+    })
+    .catch(err => {
+      return err.response.headers['location']
+    })
+
+  if (location.endsWith('latest')) {
+    return getLatestReleaseUrl(location)
+  }
+
+  return location
+}
+
+const getRepoSlug = async (moduleUrl: string): Promise<string> => {
+  const gitHttpPattern = new RegExp('https?://(.*).github.io/(.*)/?')
+  const gitSshPattern = new RegExp('git@[^:]+:(.*)/(.*)')
+
+  if (gitHttpPattern.test(moduleUrl)) {
+    const match = gitHttpPattern.exec(moduleUrl)
+    if (match) {
+      const gitOrg = match[1]
+      const repoName = match[2]
+
+      return `${gitOrg}/${repoName}`
+    }
+  } else {
+    // get information from current directory
+    const baseDir = moduleUrl || '.'
+
+    const remotes: RemoteWithRefs[] = await simpleGit({baseDir})
+      .getRemotes(true)
+      .catch(() => [])
+
+    if (!remotes || remotes.length === 0) {
+      return ''
+    }
+
+    const gitUrl = remotes[0].refs.fetch
+    const pattern = gitHttpPattern.test(gitUrl) ? gitHttpPattern : gitSshPattern
+    const match = pattern.exec(gitUrl)
+    if (match) {
+      const gitOrg = match[1]
+      const repoName = match[2].replace(new RegExp('[.]git$', ''), '')
+
+      return `${gitOrg}/${repoName}`
+    }
+  }
+
+  return ''
+}

src/commands/inputs/iascable.input.ts

@@ -17,3 +17,13 @@ export interface IascableDocsInput {
   outDir: string;
   flattenOutput: boolean;
 }
+
+export interface IascableGenerateInput {
+  module?: string;
+  moduleVersion: string;
+  repoSlug: string;
+  metadataFile: string;
+  publishBranch: string;
+  outDir: string;
+  flattenOutput: boolean;
+}

src/models/bill-of-material.model.ts

@@ -1,7 +1,7 @@
 import {default as jsYaml} from 'js-yaml';
 import {isSingleModuleVersion, Module, SingleModuleVersion} from './module.model';
 import {of, Optional} from '../util/optional';
-import {of as arrayOf} from '../util/array-util';
+import {of as arrayOf} from '../util/array-util/array-util';
 import {BillOfMaterialParsingError} from '../errors';
 
 export interface BillOfMaterialModuleDependency {

src/models/catalog.model.ts

@@ -2,7 +2,7 @@ import {LoggerApi} from '../util/logger';
 import {Container} from 'typescript-ioc';
 import {Module, ModuleDependency, ModuleProvider, ModuleVariable} from './module.model';
 import {BillOfMaterialModule} from './bill-of-material.model';
-import {of as ofArray} from '../util/array-util';
+import {of as ofArray} from '../util/array-util/array-util';
 import {Optional} from '../util/optional';
 import {findMatchingVersions} from '../util/version-resolver';
 import first from '../util/first';

src/models/module.model.ts

@@ -1,7 +1,7 @@
 import {VersionMatcher} from './version-matcher';
 import {BillOfMaterialModule} from './bill-of-material.model';
 import {isUndefined} from '../util/object-util';
-import {ArrayUtil, of as arrayOf} from '../util/array-util';
+import {ArrayUtil, of as arrayOf} from '../util/array-util/array-util';
 import {Optional} from '../util/optional';
 import {CatalogProviderModel} from './catalog.model';
 
@@ -101,6 +101,7 @@ export interface ModuleVariable {
   moduleRef?: ModuleOutputRef;
   mapper?: 'equality' | 'collect';
   important?: boolean;
+  sensitive?: boolean;
 }
 
 export interface ModuleOutputRef {
@@ -155,3 +156,10 @@ export function injectDependsOnFunction(module: Module | undefined): ModuleWithD
 
   return Object.assign({versions: [], name: ''}, module, {dependsOn})
 }
+
+export interface ModuleInterfaceModel {
+  id: string
+  name: string
+  variables?: ModuleVariable[]
+  outputs?: ModuleOutput[]
+}

src/models/stages.model.ts

@@ -9,7 +9,7 @@ import {
 import {OutputFile, OutputFileType, UrlFile} from './file.model';
 import {Module, ModuleProvider, SingleModuleVersion} from './module.model';
 import {BillOfMaterialModel, BillOfMaterialVariable} from './bill-of-material.model';
-import {ArrayUtil, of as arrayOf} from '../util/array-util';
+import {ArrayUtil, of as arrayOf} from '../util/array-util/array-util';
 import {Optional} from '../util/optional';
 import {isDefinedAndNotNull} from '../util/object-util';
 import {ModuleDocumentationApi} from '../services/module-documentation';

src/models/variables.model.ts

@@ -1,6 +1,6 @@
 import {BillOfMaterialProviderVariable} from './bill-of-material.model';
 import {ModuleVariable} from './module.model';
-import {ArrayUtil, of as arrayOf} from '../util/array-util';
+import {ArrayUtil, of as arrayOf} from '../util/array-util/array-util';
 import {isDefined} from '../util/object-util';
 import {ModuleNotFound} from '../errors';
 import {StageNotFound} from '../errors/stage-not-found.error';

src/models/version-matcher.ts

@@ -1,4 +1,4 @@
-import {of as arrayOf} from '../util/array-util';
+import {of as arrayOf} from '../util/array-util/array-util';
 import {isUndefined} from '../util/object-util';
 
 export enum VersionComparison {

src/services/bill-of-material-builder/bill-of-material-builder.ts

@@ -14,7 +14,7 @@ import {
   ModuleNotFound
 } from '../../errors';
 import {isDefinedAndNotNull, isUndefined} from '../../util/object-util';
-import {of as arrayOf} from '../../util/array-util';
+import {of as arrayOf} from '../../util/array-util/array-util';
 
 export function buildBomVariables(bomVariableYaml: string): BillOfMaterialVariable[] {
   if (!bomVariableYaml) {

src/services/dependency-graph/dependency-graph.impl.ts

@@ -7,7 +7,7 @@ import {
   SingleModuleVersion
 } from '../../models';
 import {DotGraph, DotNode} from '../../models/graph.model';
-import {of as arrayOf} from '../../util/array-util';
+import {of as arrayOf} from '../../util/array-util/array-util';
 import {Optional} from '../../util/optional';
 import {Inject} from 'typescript-ioc';
 import {CatalogLoaderApi} from '../catalog-loader';

src/services/iascable.impl.ts

@@ -16,7 +16,7 @@ import {Catalog, CatalogLoaderApi} from './catalog-loader';
 import {ModuleSelectorApi} from './module-selector';
 import {TerraformBuilderApi} from './terraform-builder';
 import {TileBuilderApi} from './tile-builder';
-import {ArrayUtil, of as arrayOf} from '../util/array-util'
+import {ArrayUtil, of as arrayOf} from '../util/array-util/array-util'
 import {Optional} from '../util/optional';
 import {DependencyGraphApi} from './dependency-graph';
 import {DotGraph, DotGraphFile} from '../models/graph.model';