From 3af6478a349f7941bf77d26b0e597cd02ff3171c Mon Sep 17 00:00:00 2001 From: Luis Aguirre Date: Fri, 22 Mar 2024 14:08:04 -0500 Subject: [PATCH 01/20] Add minimal implementation for Pipelines configure & generate CLI commands based on workspaces by using inheritance - avoiding conflicts in Config file --- bin/cortex-pipelines.js | 29 ++++++++++++++++ bin/cortex-workspaces.js | 2 +- src/commands/pipelines.js | 27 ++++++++++++++- src/commands/workspaces/configure.js | 13 ++++--- src/commands/workspaces/generate.js | 51 ++++++++++++++++++++++------ src/config.js | 8 ++++- 6 files changed, 112 insertions(+), 18 deletions(-) diff --git a/bin/cortex-pipelines.js b/bin/cortex-pipelines.js index 10eec48e..5befd596 100755 --- a/bin/cortex-pipelines.js +++ b/bin/cortex-pipelines.js @@ -7,6 +7,8 @@ import { RunPipelineCommand, DescribePipelineRunCommand, ListPipelineRunsCommand, + PipelineTemplateConfigureCommand, + PipelineGenerateCommand, } from '../src/commands/pipelines.js'; import { callCommand } from '../src/compatibility.js'; import { @@ -110,6 +112,33 @@ export function create() { return new ListPipelineRunsCommand(pipelines).execute(pipelineName, gitRepoName, options); })); + + // Configure Pipeline Template Github Repository + pipelines + .command('configure') + .option('--refresh', 'Refresh the Github access token') + .description('Configure the Cortex Template system for generating Pipeline templates') + .action((options) => { + try { + return new PipelineTemplateConfigureCommand(pipelines).execute(options); + } catch (err) { + return printError(err.message); + } + }); + + // Generate a Pipeline template + pipelines.command('generate [pipelineName] [destination]') + .option('--notree [boolean]', 'Do not dispaly generated file tree', 'false') + .option('--template ', 'Name of the template to use') + .description('Generates a folder based on a Pipeline template from the template repository') + .action((pipelineName, destination, options) => { + try { + return new PipelineGenerateCommand(pipelines).execute(pipelineName, destination, options); + } catch (err) { + return printError(err.message); + } + }); + return pipelines; } diff --git a/bin/cortex-workspaces.js b/bin/cortex-workspaces.js index dcdd0310..cd8b4883 100755 --- a/bin/cortex-workspaces.js +++ b/bin/cortex-workspaces.js @@ -2,7 +2,7 @@ import { Command } from 'commander'; import process from 'node:process'; import esMain from 'es-main'; import WorkspaceConfigureCommand from '../src/commands/workspaces/configure.js'; -import WorkspaceGenerateCommand from '../src/commands/workspaces/generate.js'; +import { WorkspaceGenerateCommand } from '../src/commands/workspaces/generate.js'; import WorkspaceBuildCommand from '../src/commands/workspaces/build.js'; import WorkspacePublishCommand from '../src/commands/workspaces/publish.js'; import { diff --git a/src/commands/pipelines.js b/src/commands/pipelines.js index dbe39670..c453b44d 100644 --- a/src/commands/pipelines.js +++ b/src/commands/pipelines.js @@ -17,12 +17,14 @@ import { printTable, printWarning, handleError, } from './utils.js'; +import WorkspaceConfigureCommand from './workspaces/configure.js'; +import { BaseGenerateCommand } from './workspaces/generate.js'; + const debug = debugSetup('cortex:cli'); dayjs.extend(relativeTime); - export const ListPipelineCommand = class { constructor(program) { this.program = program; @@ -241,3 +243,26 @@ export const ListPipelineRunsCommand = class { } } }; + +// NOTE: Easiest way to piggy-back of the existing functionality from Workspaces is to +// directly use the same logic via inheritance. Originally tried refactoring the super +// class, but the result left 2 implementations with very similar code. +// +// The constructor assigns a different configKey to avoid collision from sharing the +// same property in the config file. +export const PipelineTemplateConfigureCommand = class extends WorkspaceConfigureCommand { + constructor(program) { + super(program, 'pipelineTemplateConfig'); + } +}; + + +export const PipelineGenerateCommand = class extends BaseGenerateCommand { + constructor(program) { + super(program, 'Pipeline', 'pipelines', 'pipelinename'); + } + + async configureSubcommand() { + await (new PipelineTemplateConfigureCommand(this.program)).execute({ refresh: true }); + } +}; diff --git a/src/commands/workspaces/configure.js b/src/commands/workspaces/configure.js index 3ddb4b1f..91adae47 100644 --- a/src/commands/workspaces/configure.js +++ b/src/commands/workspaces/configure.js @@ -18,14 +18,17 @@ const GITHUB_DEVICECODE_REQUEST_URL = 'https://github.com/login/device/code'; const GITHUB_DEVICECODE_RESPONSE_URL = 'https://github.com/login/oauth/access_token'; export default class WorkspaceConfigureCommand { - constructor(program) { + constructor(program, configKey = 'templateconfig') { this.program = program; + this.configKey = configKey; } async execute(opts) { this.options = opts; try { const config = readConfig(); + const { configKey } = this; + console.log(configKey); const currentProfile = config.profiles[config.currentProfile]; console.log(`Configuring workspaces for profile ${chalk.green(config.currentProfile)}`); const answers = await inquirer.prompt([ @@ -33,13 +36,13 @@ export default class WorkspaceConfigureCommand { type: 'input', name: 'repo', message: 'Template Repository URL: ', - default: _.get(currentProfile, 'templateConfig.repo', DEFAULT_TEMPLATE_REPO), + default: _.get(currentProfile, `${configKey}.repo`, DEFAULT_TEMPLATE_REPO), }, { type: 'input', name: 'branch', message: 'Template Repository Branch:', - default: _.get(currentProfile, 'templateConfig.branch', DEFAULT_TEMPLATE_BRANCH), + default: _.get(currentProfile, `${configKey}.branch`, DEFAULT_TEMPLATE_BRANCH), }, ]); const githubToken = await validateToken(); @@ -84,7 +87,7 @@ export default class WorkspaceConfigureCommand { if (accessToken.access_token) { clearTimeout(pollTimer); persistToken(accessToken); - config.profiles[config.currentProfile].templateConfig = answers; + config.profiles[config.currentProfile][configKey] = answers; config.save(); printSuccess('\x1b[0G\x1b[2KGithub token configuration successful.', options); resolve(); @@ -125,7 +128,7 @@ export default class WorkspaceConfigureCommand { printError('\x1b[2KDevice Code request failed. Please try again.', this.options); } } else { - config.profiles[config.currentProfile].templateConfig = answers; + config.profiles[config.currentProfile][configKey] = answers; config.save(); } } catch (error) { diff --git a/src/commands/workspaces/generate.js b/src/commands/workspaces/generate.js index 79bc591a..96f73857 100644 --- a/src/commands/workspaces/generate.js +++ b/src/commands/workspaces/generate.js @@ -17,9 +17,28 @@ import { } from '../utils.js'; const METADATA_FILENAME = 'metadata.json'; -export default class WorkspaceGenerateCommand { - constructor(program) { + +export class BaseGenerateCommand { + /** + * Creates a `BaseGenerateCommand`. + * + * @param {object} program Commander program object + * @param {string} resourceName Name of the resource being generated (e.g. Skill, Pipeline, Foo, Bar) + * @param {string} resourceFolderName Name of the folder in the repository that will be + * @param {string} resourceTemplateName Expected folder name for the template - will be wrapped in underscores (e.g. __skillname__, __pipelinename__) + */ + constructor(program, resourceName, resourceFolderName, resourceTemplateName) { + // Adhoc way of implemeting an abstract class that enforces a 'configureSubcommand()' method to be present + if (new.target === BaseGenerateCommand) { + throw new TypeError('Cannot construct BaseGenerateCommand instances directly!'); + } + if (typeof this.configureSubcommand !== 'function') { + throw new TypeError('Cannot construct instance of BaseGenerateCommand without overriding async method "configureSubcommand()"!'); + } this.program = program; + this.resourceName = resourceName; + this.resourceFolderName = resourceFolderName; + this.resourceTemplateName = resourceTemplateName; } async loadTemplateTree({ repo, branch }) { @@ -88,7 +107,7 @@ export default class WorkspaceGenerateCommand { { type: 'input', name: 'name', - message: 'Enter a name for the skill:', + message: `Enter a name for the ${this.resourceName}:`, validate: (answer) => { const validation = validateName(answer); return validation.status || validation.message; @@ -113,13 +132,14 @@ export default class WorkspaceGenerateCommand { printError(this.config.profiles[this.config.currentProfile].templateConfig ? 'Github authorization is invalid. Running configuration now.\n' : 'Workspace generator is not configured. Running configuration now.\n', this.options, false); - await (new WorkspaceConfigureCommand(this.program)).execute({ refresh: true }); + await this.configureSubcommand(); this.config = readConfig(); } - if (name && fs.existsSync(path.join(destinationPath, 'skills', name))) { - printError(`Skill ${name} already exists!`, this.options); + if (name && fs.existsSync(path.join(destinationPath, this.resourceFolderName, name))) { + printError(`${this.resourceName} ${name} already exists!`, this.options); return; } + await this.loadTemplateTree(this.config.profiles[this.config.currentProfile].templateConfig); if (this.tree.length) { const template = await this.selectTemplate(options.template); @@ -127,8 +147,8 @@ export default class WorkspaceGenerateCommand { const templateFolder = path.posix.dirname(template.template.path); const templateFiles = this.globTree(`${templateFolder}/**`); const treeObj = {}; - if (fs.existsSync(path.join(destinationPath, 'skills', template.name))) { - printError(`Skill ${template.name} already exists!`, this.options); + if (fs.existsSync(path.join(destinationPath, this.resourceFolderName, template.name))) { + printError(`${this.resourceName} ${template.name} already exists!`, this.options); return; } const generatedFiles = _.map(templateFiles, (f) => { @@ -137,7 +157,7 @@ export default class WorkspaceGenerateCommand { const destFile = _.drop(f.path.split('/'), rootPathComponents.length); const targetPath = path.join(...destFile); return _.template(targetPath, { interpolate: /__([\s\S]+?)__/g })({ - skillname: generateNameFromTitle(template.name), + [this.resourceTemplateName]: generateNameFromTitle(template.name), }); } catch (err) { printError(err.message, this.options); @@ -150,7 +170,7 @@ export default class WorkspaceGenerateCommand { const fileName = path.posix.basename(f.path); if (fileName !== METADATA_FILENAME) { const templateVars = { - skillname: generateNameFromTitle(template.name), + [this.resourceTemplateName]: generateNameFromTitle(template.name), generatedFiles, template: template.template, }; @@ -183,3 +203,14 @@ export default class WorkspaceGenerateCommand { } } } + + +export class WorkspaceGenerateCommand extends BaseGenerateCommand { + constructor(program) { + super(program, 'Skill', 'skills', 'skillname'); + } + + async configureSubcommand() { + await (new WorkspaceConfigureCommand(this.program)).execute({ refresh: true }); + } +} diff --git a/src/config.js b/src/config.js index 208d7133..7b0eb0e8 100644 --- a/src/config.js +++ b/src/config.js @@ -100,6 +100,7 @@ const ProfileSchemaV5 = Joi.object().keys({ currentRegistry: Joi.string().required(), templateConfig: Joi.any().required(), featureFlags: Joi.object().optional(), + pipelineTemplateConfig: Joi.any().optional(), }); /* eslint-disable no-param-reassign */ @@ -339,7 +340,7 @@ class ConfigV4 { } class ProfileV5 { constructor(name, { - url, username, issuer, audience, jwk, project, registries, currentRegistry, templateConfig, featureFlags, + url, username, issuer, audience, jwk, project, registries, currentRegistry, templateConfig, featureFlags, pipelineTemplateConfig, }, templateConfigV4) { this.name = name; this.url = url; @@ -360,6 +361,10 @@ class ProfileV5 { repo: 'CognitiveScale/cortex-code-templates', branch: 'main', }; + this.pipelineTemplateConfig = pipelineTemplateConfig || { + repo: 'CognitiveScale/cortex-code-templates', + branch: 'main', + }; this.featureFlags = featureFlags; } @@ -383,6 +388,7 @@ class ProfileV5 { registries: this.registries, currentRegistry: this.currentRegistry, templateConfig: this.templateConfig, + pipelineTemplateConfig: this.pipelineTemplateConfig, featureFlags: this.featureFlags, // TODO: is featureFlags needed as a field? }; } From ee811ac68f411d60db5a8940faa356c7aeb97108 Mon Sep 17 00:00:00 2001 From: Luis Aguirre Date: Mon, 25 Mar 2024 17:41:54 -0500 Subject: [PATCH 02/20] WIP - Refactoring BaseGenerateCommand (trying to be more readable). Fix bug when looking up config section --- src/commands/pipelines.js | 2 +- src/commands/workspaces/generate.js | 286 +++++++++++++++++++--------- 2 files changed, 200 insertions(+), 88 deletions(-) diff --git a/src/commands/pipelines.js b/src/commands/pipelines.js index c453b44d..b7bc92a2 100644 --- a/src/commands/pipelines.js +++ b/src/commands/pipelines.js @@ -259,7 +259,7 @@ export const PipelineTemplateConfigureCommand = class extends WorkspaceConfigure export const PipelineGenerateCommand = class extends BaseGenerateCommand { constructor(program) { - super(program, 'Pipeline', 'pipelines', 'pipelinename'); + super(program, 'Pipeline', 'pipelines', 'pipelinename', 'pipelineTemplateConfig'); } async configureSubcommand() { diff --git a/src/commands/workspaces/generate.js b/src/commands/workspaces/generate.js index 96f73857..9679a5fc 100644 --- a/src/commands/workspaces/generate.js +++ b/src/commands/workspaces/generate.js @@ -5,6 +5,7 @@ import { mkdir, writeFile } from 'fs/promises'; import { minimatch } from 'minimatch'; import inquirer from 'inquirer'; import ghGot from 'gh-got'; +import debugSetup from 'debug'; import { isText } from 'istextorbinary'; import treeify from 'treeify'; import { boolean } from 'boolean'; @@ -16,6 +17,8 @@ import { printSuccess, printError, validateName, generateNameFromTitle, } from '../utils.js'; +const debug = debugSetup('cortex:cli'); + const METADATA_FILENAME = 'metadata.json'; export class BaseGenerateCommand { @@ -24,10 +27,11 @@ export class BaseGenerateCommand { * * @param {object} program Commander program object * @param {string} resourceName Name of the resource being generated (e.g. Skill, Pipeline, Foo, Bar) - * @param {string} resourceFolderName Name of the folder in the repository that will be + * @param {string} resourceFolderName Name of the folder in the repository that will be checked * @param {string} resourceTemplateName Expected folder name for the template - will be wrapped in underscores (e.g. __skillname__, __pipelinename__) + * @param {string} configKey Key in Configuration to use for fetching the underlying Template Repository, Registry, etc. */ - constructor(program, resourceName, resourceFolderName, resourceTemplateName) { + constructor(program, resourceName, resourceFolderName, resourceTemplateName, configKey) { // Adhoc way of implemeting an abstract class that enforces a 'configureSubcommand()' method to be present if (new.target === BaseGenerateCommand) { throw new TypeError('Cannot construct BaseGenerateCommand instances directly!'); @@ -39,37 +43,81 @@ export class BaseGenerateCommand { this.resourceName = resourceName; this.resourceFolderName = resourceFolderName; this.resourceTemplateName = resourceTemplateName; + this.configKey = configKey; + } + + /** + * Queries GitHub to get the Git Tree(s) in a repository. + * + * @param {string} repo Github repository identifier (e.g. `org/repo`) + * @param {string} sha Git SHA referencing the commit in the repository to retrieve + * @returns object + */ + fetchGitTree(repo, sha) { + const uri = `repos/${repo}/git/trees/${sha}?recursive=true`; + debug(`Querying Git Tree: ${uri}`); + return ghGot(uri, { + headers: { authorization: this.authorization }, + }); } - async loadTemplateTree({ repo, branch }) { - printToTerminal('\x1b[0G\x1b[2KLoading templates...'); + /** + * Loads Git Tree(s) corresponding to the HEAD of a specific branch in a git repository. + * + * @param {object} param0 Object with `repo` and `branch` keys + * @return Array - array of Git tree objects + */ + async fetchTemplateGitTrees({ repo, branch }) { + // TODO: remove the assignments to 'this' to be outside of these methods this.gitRepo = repo; this.branch = branch; this.authorization = this.githubToken; - this.tree = await ghGot(`repos/${repo}/branches/${branch || 'main'}`, { + const ghUri = `repos/${repo}/branches/${branch || 'main'}`; + debug('Loading templates from "%s" (branch = "%s"). URI: %s', repo, branch, ghUri); + const tree = await ghGot(ghUri, { headers: { authorization: this.authorization }, }) - .then((resp) => resp.body) - .then((resp) => ghGot(`repos/${repo}/git/trees/${resp.commit.sha}?recursive=true`, { - headers: { authorization: this.authorization }, - })) - .then((resp) => resp.body.tree) - .catch(() => []); - printToTerminal('\x1b[0G\x1b[2KTemplates loaded.\x1b[1E'); + .then((resp) => resp.body) + .then((resp) => this.fetchGitTree(repo, resp.commit.sha)) + .then((resp) => resp.body.tree) + .catch(() => []); + return tree; } - globTree(glob) { + /** + * Filters the Git Trees to blobs that match the given glob pattern. + * + * @param {Array} tree array of git tree objects + * @param {string} glob pattern to match + * @returns Array + */ + globTree(tree, glob) { + debug('Checking for templates that include the glob pattern: %s', glob); const filterFunc = minimatch.filter(glob, { matchBase: true }); - return _.filter(this.tree, (f) => f.type === 'blob' && filterFunc(f.path)); + return _.filter(tree, (f) => f.type === 'blob' && filterFunc(f.path)); } + /** + * Reads a file path in the remote git repository. + * + * @param {string} filePath path to the file in the repository + * @returns `Buffer` A Buffer with the contents of the file + */ async readFile(filePath) { - const response = await ghGot(`repos/${this.gitRepo}/contents/${filePath}?ref=${this.branch}`, { + const uri = `repos/${this.gitRepo}/contents/${filePath}?ref=${this.branch}`; + debug('Reading file contents "%s" from git repository (uri = "%s")', filePath, uri); + const response = await ghGot(uri, { headers: { authorization: this.authorization }, }); + debug('Successfully read file contents "%s" from git repository', filePath); return Buffer.from(response.body.content, 'base64'); } + /** + * Return a URL the Docker registry associated with the current Profile. + * + * @returns string + */ async getRegistry() { const profile = await loadProfile(this.options.profile); const registryUrl = _.get(this, 'options.registry') @@ -78,25 +126,15 @@ export class BaseGenerateCommand { return registryUrl; } - async selectTemplate(templateName) { - const registryUrl = await this.getRegistry(); - const fileNames = this.globTree(METADATA_FILENAME); - const choices = await Promise.all(_.map(fileNames, async (value) => { - const data = JSON.parse((await this.readFile(value.path)).toString()); - return { - name: data.title, - value: { ...data, path: value.path }, - }; - })); - let template; - if (templateName) { - const templateChoice = _.find(choices, { name: templateName }); - if (templateChoice) { - template = templateChoice.value; - } else { - printError(`Template ${templateName} not found!`); - } - } + /** + * Prompts the user to select the desired template. + * + * @param {string} registryUrl URL to Docker registry + * @param {Array} choices + * @param {string | null | undefined} template + * @returns object with user answers (`template`, `name`, `registry`) + */ + async promptUser(registryUrl, choices, template) { const answers = await inquirer.prompt([ { type: 'list', @@ -121,6 +159,117 @@ export class BaseGenerateCommand { return answers; } + /** + * Selects the template to clone by fetching the potential choices from the given + * Git Tree(s) and prompting the user for their desired template. + * + * @param {Array} tree Git Tree(s) describing the repository + * @param {string | null | undefined} templateName Name of template + * @returns {object} object conatining + */ + async selectTemplate(tree, templateName) { + const registryUrl = await this.getRegistry(); + debug('Docker Registry URL: %s', registryUrl); + const fileNames = this.globTree(tree, METADATA_FILENAME); + // debug('Files found matching glob pattern: %s', JSON.stringify(fileNames)); + const choices = await Promise.all(_.map(fileNames, async (value) => { + const data = JSON.parse((await this.readFile(value.path)).toString()); + return { + name: data.title, + value: { ...data, path: value.path }, + }; + })); + debug('Potential Choices: %s', JSON.stringify(choices)); + let template; + if (templateName) { + const templateChoice = _.find(choices, { name: templateName }); + if (templateChoice) { + template = templateChoice.value; + } else { + printError(`Template ${templateName} not found!`); + } + } + const answers = await this.promptUser(registryUrl, choices, template); + return answers; + } + + /** + * Checks whether the path to where the Template will be generated already exists. If so, exists the program. + * + * @param {string} destinationPath Base directory + * @param {string} name Name of the resource to be generated + */ + checkDestinationExists(destinationPath, name) { + // Check if destination already exists + if (fs.existsSync(path.join(destinationPath, this.resourceFolderName, name))) { + printError(`${this.resourceName} ${name} already exists!`, this.options); + } + } + + /** + * ?? + * + * @param {*} templateFolder + * @param {*} templateFiles + * @param {*} template + * @returns + */ + generateFiles(templateFolder, templateFiles, template) { + const generatedFiles = _.map(templateFiles, (f) => { + try { + const rootPathComponents = templateFolder.split('/'); + const destFile = _.drop(f.path.split('/'), rootPathComponents.length); + const targetPath = path.join(...destFile); + return _.template(targetPath, { interpolate: /__([\s\S]+?)__/g })({ + [this.resourceTemplateName]: generateNameFromTitle(template.name), + }); + } catch (err) { + printError(err.message, this.options); + } + return undefined; + }).join('
'); + return generatedFiles; + } + + /** + * ???? + * + * @param {*} destinationPath + * @param {*} generatedFiles + * @param {*} template + * @param {*} templateFiles + * @returns {object} + */ + async computeFileTree(destinationPath, generatedFiles, template, templateFiles) { + const treeObj = {}; + await Promise.all(_.map(templateFiles, async (f) => { + try { + const fileName = path.posix.basename(f.path); + if (fileName !== METADATA_FILENAME) { + const templateVars = { + [this.resourceTemplateName]: generateNameFromTitle(template.name), + generatedFiles, + template: template.template, + }; + let buf = await this.readFile(f.path); + /// Try not to template any non-text files. + if (isText(null, buf)) { + buf = Buffer.from(_.template(buf.toString(), { interpolate: /{{([\s\S]+?)}}/g })(templateVars)); + } + const relPath = f.path.slice(path.dirname(template.template.path).length); + const sourcePath = _.template(relPath, { interpolate: /__([\s\S]+?)__/g })(templateVars); + const targetPath = path.join(destinationPath, sourcePath); + await mkdir(path.dirname(targetPath), { recursive: true }); + await writeFile(targetPath, buf); + _.set(treeObj, sourcePath.split('/'), null); + } + } catch (err) { + printError(err.message, this.options); + } + })); + return treeObj; + } + async execute(name, destination, options) { this.options = options; this.name = name; @@ -128,68 +277,31 @@ export class BaseGenerateCommand { this.config = readConfig(); this.githubToken = await validateToken(); const destinationPath = path.resolve(destination || process.cwd()); + if (!this.githubToken) { - printError(this.config.profiles[this.config.currentProfile].templateConfig + debug('Github Token not initialized for Profile "%s" in config section "%s" - beginning configuration step', this.config.currentProfile, this.configKey); + printError(this.config.profiles[this.config.currentProfile][this.configKey] ? 'Github authorization is invalid. Running configuration now.\n' : 'Workspace generator is not configured. Running configuration now.\n', this.options, false); await this.configureSubcommand(); this.config = readConfig(); } - if (name && fs.existsSync(path.join(destinationPath, this.resourceFolderName, name))) { - printError(`${this.resourceName} ${name} already exists!`, this.options); - return; - } + this.checkDestinationExists(destinationPath, name); - await this.loadTemplateTree(this.config.profiles[this.config.currentProfile].templateConfig); - if (this.tree.length) { - const template = await this.selectTemplate(options.template); + const templateConfig = this.config.profiles[this.config.currentProfile][this.configKey]; + debug('Loading Templates - template configuration: %s', JSON.stringify(templateConfig)); + printToTerminal('\x1b[0G\x1b[2KLoading templates...'); + const tree = await this.fetchTemplateGitTrees(templateConfig); + printToTerminal('\x1b[0G\x1b[2KTemplates loaded.\x1b[1E'); + if (tree.length) { + const template = await this.selectTemplate(tree, options.template); if (template) { const templateFolder = path.posix.dirname(template.template.path); - const templateFiles = this.globTree(`${templateFolder}/**`); - const treeObj = {}; - if (fs.existsSync(path.join(destinationPath, this.resourceFolderName, template.name))) { - printError(`${this.resourceName} ${template.name} already exists!`, this.options); - return; - } - const generatedFiles = _.map(templateFiles, (f) => { - try { - const rootPathComponents = templateFolder.split('/'); - const destFile = _.drop(f.path.split('/'), rootPathComponents.length); - const targetPath = path.join(...destFile); - return _.template(targetPath, { interpolate: /__([\s\S]+?)__/g })({ - [this.resourceTemplateName]: generateNameFromTitle(template.name), - }); - } catch (err) { - printError(err.message, this.options); - } - return undefined; - }).join('
'); + const templateFiles = this.globTree(tree, `${templateFolder}/**`); + this.checkDestinationExists(destinationPath, template.name); + const generatedFiles = this.generateFiles(templateFolder, templateFiles, template); console.log(''); - await Promise.all(_.map(templateFiles, async (f) => { - try { - const fileName = path.posix.basename(f.path); - if (fileName !== METADATA_FILENAME) { - const templateVars = { - [this.resourceTemplateName]: generateNameFromTitle(template.name), - generatedFiles, - template: template.template, - }; - let buf = await this.readFile(f.path); - /// Try not to template any non-text files. - if (isText(null, buf)) { - buf = Buffer.from(_.template(buf.toString(), { interpolate: /{{([\s\S]+?)}}/g })(templateVars)); - } - const relPath = f.path.slice(path.dirname(template.template.path).length); - const sourcePath = _.template(relPath, { interpolate: /__([\s\S]+?)__/g })(templateVars); - const targetPath = path.join(destinationPath, sourcePath); - await mkdir(path.dirname(targetPath), { recursive: true }); - await writeFile(targetPath, buf); - _.set(treeObj, sourcePath.split('/'), null); - } - } catch (err) { - printError(err.message, this.options); - } - })); + const treeObj = this.computeFileTree(destinationPath, generatedFiles, template, templateFiles); if (!options || !boolean(options.notree)) { printSuccess('Generated the following files:'); console.log(''); @@ -207,7 +319,7 @@ export class BaseGenerateCommand { export class WorkspaceGenerateCommand extends BaseGenerateCommand { constructor(program) { - super(program, 'Skill', 'skills', 'skillname'); + super(program, 'Skill', 'skills', 'skillname', 'templateConfig'); } async configureSubcommand() { From c7438703fd2de4dc0d174dc0aa17e63a6561c5fa Mon Sep 17 00:00:00 2001 From: Luis Aguirre Date: Mon, 25 Mar 2024 17:58:39 -0500 Subject: [PATCH 03/20] Add comments to BaseWorkspaceGenerate command --- src/commands/workspaces/generate.js | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/src/commands/workspaces/generate.js b/src/commands/workspaces/generate.js index 9679a5fc..0ca30850 100644 --- a/src/commands/workspaces/generate.js +++ b/src/commands/workspaces/generate.js @@ -168,10 +168,11 @@ export class BaseGenerateCommand { * @returns {object} object conatining */ async selectTemplate(tree, templateName) { + // TODO(LA): Need to apply an additional filter below (when getting choices) + // to only include those which match a pre-defined resource type (e.g. 'pipeline', 'skill' <-- use as default). const registryUrl = await this.getRegistry(); debug('Docker Registry URL: %s', registryUrl); const fileNames = this.globTree(tree, METADATA_FILENAME); - // debug('Files found matching glob pattern: %s', JSON.stringify(fileNames)); const choices = await Promise.all(_.map(fileNames, async (value) => { const data = JSON.parse((await this.readFile(value.path)).toString()); return { @@ -271,6 +272,26 @@ export class BaseGenerateCommand { } async execute(name, destination, options) { + // TODO: Add some documentation (comments + mermaid diagram) showing what the heck is going on here + // + // NOTE(LA): Sequence of steps in generation process + // - read config file + // - fetch & validate existing github token + // - If the token isn't valid, then force the user to configure the template repository + // - Reread the config, after prompts + // - Check if the destination to copy to the template to exists + // - Load Git Tree(s) corresponding to the Repo/Branch + // - Query the repo to get the HEAD (SHA) for the branch + // - Query git tree(s), recursive + // - Early exit If there are NOT results for the Git Tree + // - Select the template + // - Use glob (minimatch) to list the potential template folders in the returned Git Tree. A template folder must include a `metadata.json` file + // - Read the `metadata.json` file for each potential template + // - Promp the user for the template they want (provide potential templates as choices) + // - Use glob to find the files corresponding to the template (i.e. those which need to be copied) + // - Check if the destination folder already exists (if so early exit) + // - Generate the files, using templating to substitute the desired names, etc. + // - Compute the file tree for everyting generated and display it to the user this.options = options; this.name = name; this.destination = destination; @@ -290,6 +311,7 @@ export class BaseGenerateCommand { const templateConfig = this.config.profiles[this.config.currentProfile][this.configKey]; debug('Loading Templates - template configuration: %s', JSON.stringify(templateConfig)); + // TODO: Use variable substition for theis color coding ? printToTerminal('\x1b[0G\x1b[2KLoading templates...'); const tree = await this.fetchTemplateGitTrees(templateConfig); printToTerminal('\x1b[0G\x1b[2KTemplates loaded.\x1b[1E'); From 3a54bdb43f92df25f04a21be45cdd4e45db92327 Mon Sep 17 00:00:00 2001 From: Luis Aguirre Date: Wed, 27 Mar 2024 09:43:41 -0500 Subject: [PATCH 04/20] Fix typo in README --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 11420d83..50479234 100644 --- a/README.md +++ b/README.md @@ -134,7 +134,7 @@ Executables are created for the following platforms: ## Subcommands & Compatibility Checks -The following sequene diagram shows an example running a command with Cortex CLI - namely which HTTP requests. ([This diagram can be viewered here](https://mermaid.live/view#pako:eNp9VE1z2jAQ_Ss7nh6aAeJLT54OHcZAQ4c4TEx64iLLa6IiS6ok0zCZ_PdKlg0k0Ookad--_XgrvUZUlhglkcHfDQqKU0a2mtQbAW6RxkrR1AXqcJ5QKzVMcR-OhXyBdLmAVNY1EaUJt34poi2jTBFhIQVigEpt8eU6IG-KEwQUU8iZwI4NRRk25x5zxtEcjMX60pZ7rhyFITBZLS7tmbdnq_tgcbWMxuNBmlyEB86M3YgAo5o5AsJhKUkJKy0rl0Ewpd7_lFACj7PJFD7dPdzP4ttAGlMpKrYN-BO0C5zyxp00PD0uh576F1I7BLT0to_ehsgTuFuvV_B9toa4IoVmNN5_iZmoZEDlHd-P_CGDzznqvSNdsxoht6RWQ5gjsY1GmHOyNTc99ah1WrKatUrQTksoiMESpICqc6u825ko3YhwCwtz5um2Svmyy28nuX0FDpJAiRy3xCJYCebo07Fyg-9dBk6eBGZaS53Ak9gJ-Uf003aRSSYdrfRFu0jDzAnRtGBFLCsYZ_YA6TPS3YWmVzFtR5vC53298fTcKyZKcUdmmRQmDqqPKGdnyrTlB20e3Vtj2nX3J2rjPG76lELAQXYWsWti9oFisieMk4Jj-wA7nl7UfzYlD01xz9ihL2fbDTys3j-__7ZAhWE18dduN4775zPSqKRh7rtgR6oPJRwjnbKOhlGNuiasdD_Sq7_eRPYZa9xEiduWRO820Ua8OZz_mvKDoFFidYPDqFGlG6vu9wqXb38B-i2SbA)). +The following sequence diagram shows an example running a command with Cortex CLI - namely which HTTP requests. ([This diagram can be viewered here](https://mermaid.live/view#pako:eNp9VE1z2jAQ_Ss7nh6aAeJLT54OHcZAQ4c4TEx64iLLa6IiS6ok0zCZ_PdKlg0k0Ookad--_XgrvUZUlhglkcHfDQqKU0a2mtQbAW6RxkrR1AXqcJ5QKzVMcR-OhXyBdLmAVNY1EaUJt34poi2jTBFhIQVigEpt8eU6IG-KEwQUU8iZwI4NRRk25x5zxtEcjMX60pZ7rhyFITBZLS7tmbdnq_tgcbWMxuNBmlyEB86M3YgAo5o5AsJhKUkJKy0rl0Ewpd7_lFACj7PJFD7dPdzP4ttAGlMpKrYN-BO0C5zyxp00PD0uh576F1I7BLT0to_ehsgTuFuvV_B9toa4IoVmNN5_iZmoZEDlHd-P_CGDzznqvSNdsxoht6RWQ5gjsY1GmHOyNTc99ah1WrKatUrQTksoiMESpICqc6u825ko3YhwCwtz5um2Svmyy28nuX0FDpJAiRy3xCJYCebo07Fyg-9dBk6eBGZaS53Ak9gJ-Uf003aRSSYdrfRFu0jDzAnRtGBFLCsYZ_YA6TPS3YWmVzFtR5vC53298fTcKyZKcUdmmRQmDqqPKGdnyrTlB20e3Vtj2nX3J2rjPG76lELAQXYWsWti9oFisieMk4Jj-wA7nl7UfzYlD01xz9ihL2fbDTys3j-__7ZAhWE18dduN4775zPSqKRh7rtgR6oPJRwjnbKOhlGNuiasdD_Sq7_eRPYZa9xEiduWRO820Ua8OZz_mvKDoFFidYPDqFGlG6vu9wqXb38B-i2SbA)). The Cortex CLI will limit the set of available subcommands, to those which are supported by the Cortex cluster that it is communicating with. From 8cf915b5fdf5c637da3af65135ef47ad3483bb69 Mon Sep 17 00:00:00 2001 From: Luis Aguirre Date: Wed, 27 Mar 2024 09:45:00 -0500 Subject: [PATCH 05/20] Reorganize WorkspaceConfigureCommand to have a shared super class with PipelinesConfigureCommand --- bin/cortex-workspaces.js | 2 +- src/commands/workspaces/configure.js | 26 ++++++++++++++++++++++---- src/commands/workspaces/generate.js | 2 +- 3 files changed, 24 insertions(+), 6 deletions(-) diff --git a/bin/cortex-workspaces.js b/bin/cortex-workspaces.js index cd8b4883..1ebc6817 100755 --- a/bin/cortex-workspaces.js +++ b/bin/cortex-workspaces.js @@ -1,7 +1,7 @@ import { Command } from 'commander'; import process from 'node:process'; import esMain from 'es-main'; -import WorkspaceConfigureCommand from '../src/commands/workspaces/configure.js'; +import { WorkspaceConfigureCommand } from '../src/commands/workspaces/configure.js'; import { WorkspaceGenerateCommand } from '../src/commands/workspaces/generate.js'; import WorkspaceBuildCommand from '../src/commands/workspaces/build.js'; import WorkspacePublishCommand from '../src/commands/workspaces/publish.js'; diff --git a/src/commands/workspaces/configure.js b/src/commands/workspaces/configure.js index 91adae47..2dd709af 100644 --- a/src/commands/workspaces/configure.js +++ b/src/commands/workspaces/configure.js @@ -17,10 +17,23 @@ const DEFAULT_TEMPLATE_BRANCH = 'main'; const GITHUB_DEVICECODE_REQUEST_URL = 'https://github.com/login/device/code'; const GITHUB_DEVICECODE_RESPONSE_URL = 'https://github.com/login/oauth/access_token'; -export default class WorkspaceConfigureCommand { - constructor(program, configKey = 'templateconfig') { +export class BaseConfigureCommand { + /** + * Creates a Command object that prompts the user to configures a remote Github Repository & Branch + * as the source of for templates. + * + * @param {object} program Commander progrma object + * @param {string} configKey Key in user Config file to store the configuration settings under + * @param {string} context String context for what is being configured - e.g. 'Pipelines', 'Workspaces' + */ + constructor(program, configKey, context) { + // Adhoc way of implemeting an abstract class + if (new.target === BaseConfigureCommand) { + throw new TypeError('Cannot construct BaseConfigureCommand instances directly!'); + } this.program = program; this.configKey = configKey; + this.context = context; } async execute(opts) { @@ -28,9 +41,8 @@ export default class WorkspaceConfigureCommand { try { const config = readConfig(); const { configKey } = this; - console.log(configKey); const currentProfile = config.profiles[config.currentProfile]; - console.log(`Configuring workspaces for profile ${chalk.green(config.currentProfile)}`); + console.log(`Configuring ${this.context} for profile ${chalk.green(config.currentProfile)}`); const answers = await inquirer.prompt([ { type: 'input', @@ -136,3 +148,9 @@ export default class WorkspaceConfigureCommand { } } } + +export class WorkspaceConfigureCommand extends BaseConfigureCommand { + constructor(program) { + super(program, 'templateConfig', 'workspaces'); + } +} diff --git a/src/commands/workspaces/generate.js b/src/commands/workspaces/generate.js index 0ca30850..958baa60 100644 --- a/src/commands/workspaces/generate.js +++ b/src/commands/workspaces/generate.js @@ -10,7 +10,7 @@ import { isText } from 'istextorbinary'; import treeify from 'treeify'; import { boolean } from 'boolean'; import _ from 'lodash'; -import WorkspaceConfigureCommand from './configure.js'; +import { WorkspaceConfigureCommand } from './configure.js'; import { readConfig, loadProfile } from '../../config.js'; import { printToTerminal, validateToken } from './workspace-utils.js'; import { From 73b79363726c8da98ab4d361d88eed32d518e7db Mon Sep 17 00:00:00 2001 From: Luis Aguirre Date: Wed, 27 Mar 2024 11:47:45 -0500 Subject: [PATCH 06/20] Rename base template configure/generate commands. Refactor generation logic into smaller functions with a custom type --- src/commands/pipelines.js | 14 +- src/commands/workspaces/configure.js | 8 +- src/commands/workspaces/generate.js | 220 +++++++++++++++++---------- 3 files changed, 147 insertions(+), 95 deletions(-) diff --git a/src/commands/pipelines.js b/src/commands/pipelines.js index b7bc92a2..754ed215 100644 --- a/src/commands/pipelines.js +++ b/src/commands/pipelines.js @@ -17,8 +17,8 @@ import { printTable, printWarning, handleError, } from './utils.js'; -import WorkspaceConfigureCommand from './workspaces/configure.js'; -import { BaseGenerateCommand } from './workspaces/generate.js'; +import { TemplateConfigureCommand } from './workspaces/configure.js'; +import { TemplateGenerationCommand } from './workspaces/generate.js'; const debug = debugSetup('cortex:cli'); @@ -245,19 +245,17 @@ export const ListPipelineRunsCommand = class { }; // NOTE: Easiest way to piggy-back of the existing functionality from Workspaces is to -// directly use the same logic via inheritance. Originally tried refactoring the super -// class, but the result left 2 implementations with very similar code. +// directly use the same logic via inheritance. // // The constructor assigns a different configKey to avoid collision from sharing the // same property in the config file. -export const PipelineTemplateConfigureCommand = class extends WorkspaceConfigureCommand { +export const PipelineTemplateConfigureCommand = class extends TemplateConfigureCommand { constructor(program) { - super(program, 'pipelineTemplateConfig'); + super(program, 'pipelineTemplateConfig', 'Pipelines'); } }; - -export const PipelineGenerateCommand = class extends BaseGenerateCommand { +export const PipelineGenerateCommand = class extends TemplateGenerationCommand { constructor(program) { super(program, 'Pipeline', 'pipelines', 'pipelinename', 'pipelineTemplateConfig'); } diff --git a/src/commands/workspaces/configure.js b/src/commands/workspaces/configure.js index 2dd709af..16bf574d 100644 --- a/src/commands/workspaces/configure.js +++ b/src/commands/workspaces/configure.js @@ -17,7 +17,7 @@ const DEFAULT_TEMPLATE_BRANCH = 'main'; const GITHUB_DEVICECODE_REQUEST_URL = 'https://github.com/login/device/code'; const GITHUB_DEVICECODE_RESPONSE_URL = 'https://github.com/login/oauth/access_token'; -export class BaseConfigureCommand { +export class TemplateConfigureCommand { /** * Creates a Command object that prompts the user to configures a remote Github Repository & Branch * as the source of for templates. @@ -28,8 +28,8 @@ export class BaseConfigureCommand { */ constructor(program, configKey, context) { // Adhoc way of implemeting an abstract class - if (new.target === BaseConfigureCommand) { - throw new TypeError('Cannot construct BaseConfigureCommand instances directly!'); + if (new.target === TemplateConfigureCommand) { + throw new TypeError('Cannot construct TemplateConfigureCommand instances directly!'); } this.program = program; this.configKey = configKey; @@ -149,7 +149,7 @@ export class BaseConfigureCommand { } } -export class WorkspaceConfigureCommand extends BaseConfigureCommand { +export class WorkspaceConfigureCommand extends TemplateConfigureCommand { constructor(program) { super(program, 'templateConfig', 'workspaces'); } diff --git a/src/commands/workspaces/generate.js b/src/commands/workspaces/generate.js index 958baa60..020dfa39 100644 --- a/src/commands/workspaces/generate.js +++ b/src/commands/workspaces/generate.js @@ -21,7 +21,32 @@ const debug = debugSetup('cortex:cli'); const METADATA_FILENAME = 'metadata.json'; -export class BaseGenerateCommand { +/** + * Class wrapping the raw results from fetching a Git Tree. + * + * (Primarily meant for type hinting). + */ +export class GitTreeWrapper { + constructor({ + // eslint-disable-next-line no-shadow + path, mode, type, sha, size, url, + }) { + this.path = path; + this.mode = mode; + this.type = type; + this.sha = sha; + this.size = size; + this.url = url; + if (!this.path) { + throw TypeError('Raw Git Tree is missing attribute "path"'); + } + if (!this.type) { + throw TypeError('Raw Git Tree is missing attribute "type"'); + } + } +} + +export class TemplateGenerationCommand { /** * Creates a `BaseGenerateCommand`. * @@ -33,25 +58,57 @@ export class BaseGenerateCommand { */ constructor(program, resourceName, resourceFolderName, resourceTemplateName, configKey) { // Adhoc way of implemeting an abstract class that enforces a 'configureSubcommand()' method to be present - if (new.target === BaseGenerateCommand) { - throw new TypeError('Cannot construct BaseGenerateCommand instances directly!'); + if (new.target === TemplateGenerationCommand) { + throw new TypeError('Cannot construct TemplateGenerationCommand instances directly!'); } if (typeof this.configureSubcommand !== 'function') { - throw new TypeError('Cannot construct instance of BaseGenerateCommand without overriding async method "configureSubcommand()"!'); + throw new TypeError('Cannot construct instance of TemplateGenerationCommand without overriding async method "configureSubcommand()"!'); } + // Set options controlling direct behavior this.program = program; this.resourceName = resourceName; this.resourceFolderName = resourceFolderName; this.resourceTemplateName = resourceTemplateName; this.configKey = configKey; } + + /** + * Initializes the `BaseGenerateCommand` by setting execution-time options and retrieving the template + * configuration from the config file. + * + * @param {string} name Desired name for the generated template + * @param {string} destination Path to where the template should be generated + * @param {option} options CLI runtime options + */ + async init(name, destination, options) { + this.options = options; + this.name = name; + this.destination = destination; + this.config = readConfig(); + this.githubToken = await validateToken(); + this.authorization = this.githubToken; + const templateConfig = this.getTemplateConfig(); + this.gitRepo = templateConfig.repo; + this.branch = templateConfig.branch; + debug('Initialized template configuration. Raw configuration: %s', JSON.stringify(templateConfig)); + debug('GitHub Token Set? %s', this.githubToken != null); + } + + /** + * Returns the configuration for the template repository/branch. + * + * @returns {object} The configuraiton for the template repository + */ + getTemplateConfig() { + return this.config.profiles[this.config.currentProfile][this.configKey]; + } /** - * Queries GitHub to get the Git Tree(s) in a repository. + * Queries GitHub to get the Git Tree(s) from a repository. * * @param {string} repo Github repository identifier (e.g. `org/repo`) * @param {string} sha Git SHA referencing the commit in the repository to retrieve - * @returns object + * @returns {object} object returned by the GitHub API */ fetchGitTree(repo, sha) { const uri = `repos/${repo}/git/trees/${sha}?recursive=true`; @@ -62,34 +119,36 @@ export class BaseGenerateCommand { } /** - * Loads Git Tree(s) corresponding to the HEAD of a specific branch in a git repository. + * Loads Git Tree(s) corresponding to the HEAD of configured repo/branch. * - * @param {object} param0 Object with `repo` and `branch` keys - * @return Array - array of Git tree objects + * @return {Array} array of Git tree objects */ - async fetchTemplateGitTrees({ repo, branch }) { - // TODO: remove the assignments to 'this' to be outside of these methods - this.gitRepo = repo; - this.branch = branch; - this.authorization = this.githubToken; - const ghUri = `repos/${repo}/branches/${branch || 'main'}`; - debug('Loading templates from "%s" (branch = "%s"). URI: %s', repo, branch, ghUri); - const tree = await ghGot(ghUri, { + async fetchGitTreeFromBranch() { + const ghUri = `repos/${this.gitRepo}/branches/${this.branch || 'main'}`; + debug('Loading templates from "%s" (branch = "%s"). URI: %s', this.gitRepo, this.branch, ghUri); + const options = { headers: { authorization: this.authorization }, - }) + }; + const tree = await ghGot(ghUri, options) .then((resp) => resp.body) - .then((resp) => this.fetchGitTree(repo, resp.commit.sha)) + .then((resp) => this.fetchGitTree(this.gitRepo, resp.commit.sha)) .then((resp) => resp.body.tree) - .catch(() => []); + .then((resp) => resp.map((t) => new GitTreeWrapper(t))) + .catch((e) => { + debug('Error encountered while trying to fetch the git tree for the repository. %s', e); + if (e?.response?.statusCode === 404) { + printError('Unable to retrieve templates. Repository or branch not found!', this.options); + } + }); return tree; } /** - * Filters the Git Trees to blobs that match the given glob pattern. + * Filters the Git Trees to only those that are blobs that match the glob pattern. * - * @param {Array} tree array of git tree objects + * @param {Array} tree array of git tree objects * @param {string} glob pattern to match - * @returns Array + * @returns {Array} instances matching the glob pattern */ globTree(tree, glob) { debug('Checking for templates that include the glob pattern: %s', glob); @@ -98,7 +157,7 @@ export class BaseGenerateCommand { } /** - * Reads a file path in the remote git repository. + * Reads a file in the configured remote git repository. * * @param {string} filePath path to the file in the repository * @returns `Buffer` A Buffer with the contents of the file @@ -132,7 +191,7 @@ export class BaseGenerateCommand { * @param {string} registryUrl URL to Docker registry * @param {Array} choices * @param {string | null | undefined} template - * @returns object with user answers (`template`, `name`, `registry`) + * @returns {object} object with user answers (`template`, `name`, `registry`) */ async promptUser(registryUrl, choices, template) { const answers = await inquirer.prompt([ @@ -163,13 +222,14 @@ export class BaseGenerateCommand { * Selects the template to clone by fetching the potential choices from the given * Git Tree(s) and prompting the user for their desired template. * - * @param {Array} tree Git Tree(s) describing the repository + * @param {Array} tree Git Trees for the contents of repository * @param {string | null | undefined} templateName Name of template - * @returns {object} object conatining + * @returns {object} object with user selections (`template`, `name`, `registry`) */ async selectTemplate(tree, templateName) { // TODO(LA): Need to apply an additional filter below (when getting choices) // to only include those which match a pre-defined resource type (e.g. 'pipeline', 'skill' <-- use as default). + // TODO(LA): Should only consider template choices that have 'enabled == true' const registryUrl = await this.getRegistry(); debug('Docker Registry URL: %s', registryUrl); const fileNames = this.globTree(tree, METADATA_FILENAME); @@ -201,28 +261,28 @@ export class BaseGenerateCommand { * @param {string} name Name of the resource to be generated */ checkDestinationExists(destinationPath, name) { - // Check if destination already exists + // Fail if destination already exists if (fs.existsSync(path.join(destinationPath, this.resourceFolderName, name))) { printError(`${this.resourceName} ${name} already exists!`, this.options); } } /** - * ?? + * Applies templating over the filenames in the set of files to generate. * - * @param {*} templateFolder - * @param {*} templateFiles - * @param {*} template - * @returns + * @param {string} templateFolder Folder name for the template to be generated + * @param {Array} templateFiles Files that will be templated + * @param {string} templateName String name to apply to the template + * @returns {string} A string with all the filenames to generate separated by `
` */ - generateFiles(templateFolder, templateFiles, template) { + applyTemplatingToFilenames(templateFolder, templateFiles, templateName) { const generatedFiles = _.map(templateFiles, (f) => { try { const rootPathComponents = templateFolder.split('/'); const destFile = _.drop(f.path.split('/'), rootPathComponents.length); const targetPath = path.join(...destFile); return _.template(targetPath, { interpolate: /__([\s\S]+?)__/g })({ - [this.resourceTemplateName]: generateNameFromTitle(template.name), + [this.resourceTemplateName]: generateNameFromTitle(templateName), }); } catch (err) { printError(err.message, this.options); @@ -233,15 +293,16 @@ export class BaseGenerateCommand { } /** - * ???? + * Generates the selected template by templating its contents and writing its content to disk. + * Returns the file tree with what files were created. * - * @param {*} destinationPath - * @param {*} generatedFiles - * @param {*} template - * @param {*} templateFiles - * @returns {object} + * @param {string} destinationPath path to generate template at + * @param {string} generatedFiles String with all the filenames that will be created + * @param {object} template Object with `template`, `name`, and `registry` + * @param {Array} templateFiles Array of objects representing files to generate + * @returns {object} An object containing the file tree for all generated files */ - async computeFileTree(destinationPath, generatedFiles, template, templateFiles) { + async generateTemplatedFiles(destinationPath, generatedFiles, template, templateFiles) { const treeObj = {}; await Promise.all(_.map(templateFiles, async (f) => { try { @@ -271,66 +332,59 @@ export class BaseGenerateCommand { return treeObj; } + /** + * Generates the specified template at the given destination. + * + * @param {Array Date: Wed, 27 Mar 2024 11:50:09 -0500 Subject: [PATCH 07/20] Add missing await that caused no file tree to be returned --- src/commands/workspaces/generate.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/commands/workspaces/generate.js b/src/commands/workspaces/generate.js index 020dfa39..ca6bca4b 100644 --- a/src/commands/workspaces/generate.js +++ b/src/commands/workspaces/generate.js @@ -376,7 +376,7 @@ export class TemplateGenerationCommand { if (tree.length) { const template = await this.selectTemplate(tree, options.template); if (template) { - const treeObj = this.generateTemplate(tree, template, destinationPath); + const treeObj = await this.generateTemplate(tree, template, destinationPath); console.log(''); if (!options || !boolean(options.notree)) { printSuccess('Generated the following files:'); From 2406edaadbd5345d129b4ffb3798082c5c923f55 Mon Sep 17 00:00:00 2001 From: Luis Aguirre Date: Wed, 27 Mar 2024 11:57:16 -0500 Subject: [PATCH 08/20] Add variables for ANSI escape sequences to reduce clutter --- src/commands/workspaces/generate.js | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/commands/workspaces/generate.js b/src/commands/workspaces/generate.js index ca6bca4b..6e3a8292 100644 --- a/src/commands/workspaces/generate.js +++ b/src/commands/workspaces/generate.js @@ -21,6 +21,11 @@ const debug = debugSetup('cortex:cli'); const METADATA_FILENAME = 'metadata.json'; +// ANSI escape sequences for pretty printing messages during generation. +// Reference: https://gist.github.com/fnky/458719343aabd01cfb17a3a4f7296797#cursor-controls +const RESET_CURSOR_ANSI_ESCAPE = '\x1b[0G\x1b[2K'; +const CURSOR_NEXT_LINE_ANSI_ESCAPE = '\x1b[1E'; + /** * Class wrapping the raw results from fetching a Git Tree. * @@ -369,10 +374,9 @@ export class TemplateGenerationCommand { await this.init(name, destination, options); } this.checkDestinationExists(destinationPath, name); - // TODO: Use variable substition for this color coding ? - printToTerminal('\x1b[0G\x1b[2KLoading templates...'); + printToTerminal(`${RESET_CURSOR_ANSI_ESCAPE}Loading templates...`); const tree = await this.fetchGitTreeFromBranch(); - printToTerminal('\x1b[0G\x1b[2KTemplates loaded.\x1b[1E'); + printToTerminal(`${RESET_CURSOR_ANSI_ESCAPE}Templates loaded.${CURSOR_NEXT_LINE_ANSI_ESCAPE}`); if (tree.length) { const template = await this.selectTemplate(tree, options.template); if (template) { From 2158b0a1bdda53c947bec62d8c72c95e1b89f74c Mon Sep 17 00:00:00 2001 From: Luis Aguirre Date: Wed, 27 Mar 2024 12:00:46 -0500 Subject: [PATCH 09/20] Use printToTerminal() instead of console.log() for all output --- src/commands/workspaces/generate.js | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/commands/workspaces/generate.js b/src/commands/workspaces/generate.js index 6e3a8292..61396462 100644 --- a/src/commands/workspaces/generate.js +++ b/src/commands/workspaces/generate.js @@ -381,12 +381,12 @@ export class TemplateGenerationCommand { const template = await this.selectTemplate(tree, options.template); if (template) { const treeObj = await this.generateTemplate(tree, template, destinationPath); - console.log(''); + printToTerminal(''); if (!options || !boolean(options.notree)) { printSuccess('Generated the following files:'); - console.log(''); - console.log(destinationPath); - console.log(treeify.asTree(treeObj, true)); + printToTerminal(''); + printToTerminal(destinationPath); + printToTerminal(treeify.asTree(treeObj, true)); } printSuccess('Generation complete.', this.options); } From ad754c0ec2e39d9ddea5cfc63ed5c2213a901004 Mon Sep 17 00:00:00 2001 From: Luis Aguirre Date: Wed, 27 Mar 2024 17:15:00 -0500 Subject: [PATCH 10/20] Apply enabled & resourceType filters for templates matching the specified metadata.json --- src/commands/workspaces/generate.js | 26 +++++++++++++++++--------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/src/commands/workspaces/generate.js b/src/commands/workspaces/generate.js index 61396462..d1cc1bd5 100644 --- a/src/commands/workspaces/generate.js +++ b/src/commands/workspaces/generate.js @@ -232,20 +232,28 @@ export class TemplateGenerationCommand { * @returns {object} object with user selections (`template`, `name`, `registry`) */ async selectTemplate(tree, templateName) { - // TODO(LA): Need to apply an additional filter below (when getting choices) - // to only include those which match a pre-defined resource type (e.g. 'pipeline', 'skill' <-- use as default). - // TODO(LA): Should only consider template choices that have 'enabled == true' - const registryUrl = await this.getRegistry(); - debug('Docker Registry URL: %s', registryUrl); - const fileNames = this.globTree(tree, METADATA_FILENAME); - const choices = await Promise.all(_.map(fileNames, async (value) => { + // Utils for proessing the Metadata file + const loadMetadata = async (value) => { const data = JSON.parse((await this.readFile(value.path)).toString()); return { name: data.title, value: { ...data, path: value.path }, }; - })); - debug('Potential Choices: %s', JSON.stringify(choices)); + }; + const filterByEnabled = (data) => data.value?.enabled; + const filterByResourceType = (data) => (data.value?.resourceType) === this.resourceName; + + // Find possible choices for the template selection + const registryUrl = await this.getRegistry(); + debug('Docker Registry URL: %s', registryUrl); + const fileNames = this.globTree(tree, METADATA_FILENAME); + const choices = (await Promise.all(_.map(fileNames, loadMetadata))) + .filter(filterByEnabled) + .filter(filterByResourceType); + debug(JSON.stringify(choices)); + debug('Potential Choices: %s', JSON.stringify(choices.map((c) => c.name))); + + // Prompt user for selected template (handle template specified up front) let template; if (templateName) { const templateChoice = _.find(choices, { name: templateName }); From 41821826033dd99380a7c745357b4beeb22cf5eb Mon Sep 17 00:00:00 2001 From: Luis Aguirre Date: Wed, 27 Mar 2024 17:26:43 -0500 Subject: [PATCH 11/20] Add temporary docs folder - maybe not appropriate for public repo --- docs/workspaces.md | 236 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 236 insertions(+) create mode 100644 docs/workspaces.md diff --git a/docs/workspaces.md b/docs/workspaces.md new file mode 100644 index 00000000..30448bcf --- /dev/null +++ b/docs/workspaces.md @@ -0,0 +1,236 @@ +# Workspaces + +## Overview + +Note that `workspaces` CLI commands are narrowly focused on generating Skill templates. Ideally `workspaces` would encompass Pipeline as well, but the initial implementation for Pipelines uses its CLI commands (similar to `workspaces`). + +Both commands only support Git Repositories hosted on GitHub! + +### Templates + +The primary repository with templates can be found at: + +* A template is any folder within the git Repository that contains a `metadata.json` file. + + Expected structure for `metadata.json`: + + ```json + { + "name": "", + "title": "", + "description": "", + "tags": ["list", "of", "tags", "applied", "to", "template"], + "enabled": true, + "resourceType": "Skill" // e.g. "Pipeline" + } + ``` + +* Templates can be nested in any subfolder in the repository + +### Additional Reference + +* [Example Cortex Development Workflow](https://drive.google.com/file/d/1tPyuqtNFz9JFtJuQE6HRxou_SLKIK8fO/view?usp=drive_link) +* [Relation to Skill Building](https://docs.google.com/presentation/d/1k4vJ7d5oGbvFaUezl5dBXHtKc-CZK8IAGlvIq-RpJrk/edit#slide=id.g12c69d77b32_0_42) (i.e. migration from traditional Skill Building) + +## Workspaces Configure + +The configuration process implements the [Github OAuth Device Flow](https://docs.github.com/en/apps/oauth-apps/building-oauth-apps/authorizing-oauth-apps#device-flow). The reason for implementing this flow is primarily to avoid rate limiting issues from the GitHub rest API, see: + +* [Unauthenticated User Rate Limiting](https://docs.github.com/en/rest/using-the-rest-api/rate-limits-for-the-rest-api?apiVersion=2022-11-28#primary-rate-limit-for-unauthenticated-users) (~60 requests per hour) +* [Authenticated User Rate Limiting](https://docs.github.com/en/rest/using-the-rest-api/rate-limits-for-the-rest-api?apiVersion=2022-11-28#primary-rate-limit-for-authenticated-users) (~5000 requests per hour) + +The GitHub authentication does NOT allow users to access private repositories. Using a private repository would require [registering the Cortex CLI App in your developer settings](https://github.com/orgs/community/discussions/48102). + +```mermaid +%% Title: Cortex Workspaces configure CLI command +sequenceDiagram + autonumber + title Workspaces Configure CLI subcommand + Actor D as Dev + participant C as Cortex CLI + participant F as Filesystem + + %% Mac: KeyChain + %% Linux: Secret Service API/libsecret + %% Windows: Credential Vault + participant K as System Keychain
(Optional) + + participant B as Browser + participant G as GitHub + + D->>+C: Configure CLI + Note over D,C: cortex workspaces configure --refresh --no-timeout + + C->>+F: Read Config File
fetch existing config + C->>+D: Prompt user for Repository/Branch + C->>+D: User responds with answers + + %% TODO: Check if token is valid + %% If it's invalid and not being refreshed - exit with error + + C->>+B: Browser Opens at GitHub + D->>+B: Enter code & authorize Cortex CLI + loop Fixed Interval (e.g. 5 seconds) + C->>+G: Fetch GitHub Device Code + G->>+C: JSON (Potential Device Code) + Note over C,G: HHTP POST https://github.com/login/device/code + + critical Check Access Token + option Access Token returned + alt Keychain service available + C->>+K: Store token in Keychain + else Local File fallback + C->>+F: Store token as file in Config folder + end + C->>+D: Configuration Successful + option Authorization Pending + C->>+C: Update Expiration Time + option Slow Down + C->>+C: Increment Poll Interval + option Expired Token + C->>+D: Exit with error - re-configure and try again + option Incorrect Device Code + C->>+D: Exit with error - incorrect device code entered + option Access Denied + C->>+D: Exit with error - Access denied by user + option Unexpected Error + C->>+D: Exit with error + end + end +``` + + + +## Workspaces Generate + +The following diagram shows the sequence of steps taken when generating a template. + +```mermaid +sequenceDiagram + autonumber + title Workspaces Generate CLI subcommand + Actor D as Dev + participant C as Cortex CLI + participant F as Filesystem + + %% Mac: KeyChain + %% Linux: Secret Service API/libsecret + %% Windows: Credential Vault + participant K as System Keychain
(Optional) + participant G as GitHub + + D->>+C: Generate Pipeline + Note over D,C: cortex workspaces generate + + critical Github Token Validation + C->>+F: Read the user config File + alt Keychain service available + C->>+K: Load token from Keychain + else Local File fallback + C->>+F: Load token from cached file in Config folder + end + %% Valdiate the Token + C->>+G: Validate Token with Github API + Note over C,G: HTTP GET https://api.github.com/user (with Token) + + option Token is Invalid + C->>+C: Force user to configure template repository (recreate Token) + option Token is Valid + C->>+C: Continue (No-Op) + end + + opt Fail early if destination already exists + C->>+F: Check if the destination exists + C->>+D: Report Error to user + end + + critical Fetch Git Tree for the Repository/Branch + C->>+G: Fetch the HEAD of the configured repository/branch + G->>+C: git SHA (JSON) + Note over C,G: HTTP GET https://api.github.com/repos//branches/ + alt Repository & Branch exist + C->>+G: Fetch Git Tree(s) from the Repo + G->>+C: Git Tree (JSON)/fetch + Note over C,G: HTTP GET https://api.github.com/repos//git/trees/?recursive=true + end + end + + critical Select Template + Note right of C: Templates are identifed based
on presence of `metadata.json` + C->>+C: Use glob to find potential templates in Git Tree + loop For each potential template + C->>+G: Read the metadata.json file for the Template + G->>+C: JSON + Note over C,G: HTTP GET https://api.github.com/repos//contents/?ref= + end + + C->>+D: Prompt User for template selection + D->>+C: Answers (Template Choice, Template Name, etc.) + alt No Templates found + C->>+D: Display Error + end + alt Selected Template does not exist + C->>+D: Display Error + end + end + + critical Generate Template + C->>+C: Use glob to identify files corresponding to template in Git Tree + + opt Fail early if destination already exists + C->>+F: Check if the destination exists + C->>+D: Report Error to user + end + + C->>+C: Compute generated files
with templated values + loop For each templated file + C->>+G: Fetch file contents + G->>+C: File Contents (stream) + Note over C,G: HTTP GET https://api.github.com/repos//contents/?ref= + C->>+C: Apply templating + C->>+F: Write templated file to Filesystem + end + end + + opt Display generated file tree to user + C->>+D: Print the generated file tree + end +``` + +## Relation to Pipelines + +The Cortex CLI offers similar capabilities for generating Pipelines template, similar to Workspaces, however, the both are distinct features (subcommands) in the CLI. + +### Pipelines Configure + +The process is simlilar to that of [workspaces configure](#workspaces-configure), but: + +* A separate section of the Config file is used to store the Repository configuration - i.e. The Pipeline Template repository can be different + +```bash +$ cortex pipelines configure --refresh --no-timeout +Configuring workspaces for profile qa-aks +? Template Repository URL: CognitiveScale/cortex-code-templates +? Template Repository Branch: FAB-6046-pipeline-generate +Opening browser at https://github.com/login/device +Please enter the following code to authorize the Cortex CLI: FB05-0378 ( Expires in 14 minutes and 58 seconds ) - CTRL-C to abort +Github token configuration successful. +``` + +### Pipelines Generate + +The process is simlilar to that of [workspaces generate](#workspaces-generate), but only Pipeline templates are available. + +```bash +$ cortex pipelines generate +``` + +## Workspaces vs Pipelines + +The Cortex CLI offers similar capabilities for generating Pipelines template, similar to Workspaces, however, the both are distinct features (subcommands) in the CLI. From 7d48f8702b306d62c45bfb3ffc74cc96b799d251 Mon Sep 17 00:00:00 2001 From: Luis Aguirre Date: Wed, 27 Mar 2024 18:09:27 -0500 Subject: [PATCH 12/20] Add TOOD and note about templating --- docs/workspaces.md | 1 + src/commands/workspaces/generate.js | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/workspaces.md b/docs/workspaces.md index 30448bcf..344a6942 100644 --- a/docs/workspaces.md +++ b/docs/workspaces.md @@ -26,6 +26,7 @@ The primary repository with templates can be found at: { + // TODO: this should have error handling to avoid a bad template crashing the CLI const data = JSON.parse((await this.readFile(value.path)).toString()); return { name: data.title, From 36f031c37b7369ff66be6af978035c216fdd3f4a Mon Sep 17 00:00:00 2001 From: Luis Aguirre Date: Wed, 27 Mar 2024 19:45:56 -0500 Subject: [PATCH 13/20] Add debugging logs to the Generate command, and add ability to filter filenames for Pipelines - dbt (jinja2) clashes with lodash --- src/commands/pipelines.js | 13 ++++++++ src/commands/workspaces/generate.js | 47 ++++++++++++++++++++++++++--- 2 files changed, 55 insertions(+), 5 deletions(-) diff --git a/src/commands/pipelines.js b/src/commands/pipelines.js index 754ed215..860ac4e4 100644 --- a/src/commands/pipelines.js +++ b/src/commands/pipelines.js @@ -1,6 +1,7 @@ import debugSetup from 'debug'; import fs from 'fs'; import dayjs from 'dayjs'; +import path from 'path'; import _ from 'lodash'; import relativeTime from 'dayjs/plugin/relativeTime.js'; import { loadProfile } from '../config.js'; @@ -260,6 +261,18 @@ export const PipelineGenerateCommand = class extends TemplateGenerationCommand { super(program, 'Pipeline', 'pipelines', 'pipelinename', 'pipelineTemplateConfig'); } + // TODO: Need a better way to handle this filtering + filterByFileName(filepath) { + // NOTE: It is common for dbt's templating (jinja2?) to collide with Lodash's templating syntax. + // Current workaround is to exclude DBT & SQL files from templating, only downside is that those + // can't use the {{pipelinename}} variable. + const fileName = path.posix.basename(filepath); + if (path.extname(fileName) === '.sql' || filepath.includes('dbt_packages') || filepath.includes('/dbt/')) { + return true; + } + return false; + } + async configureSubcommand() { await (new PipelineTemplateConfigureCommand(this.program)).execute({ refresh: true }); } diff --git a/src/commands/workspaces/generate.js b/src/commands/workspaces/generate.js index 8509842d..e41ec21d 100644 --- a/src/commands/workspaces/generate.js +++ b/src/commands/workspaces/generate.js @@ -69,6 +69,9 @@ export class TemplateGenerationCommand { if (typeof this.configureSubcommand !== 'function') { throw new TypeError('Cannot construct instance of TemplateGenerationCommand without overriding async method "configureSubcommand()"!'); } + if (typeof this.filterByFileName !== 'function') { + throw new TypeError('Cannot construct instance of TemplateGenerationCommand without overriding async method "filterByFileName(string) -> bool"!'); + } // Set options controlling direct behavior this.program = program; this.resourceName = resourceName; @@ -158,7 +161,9 @@ export class TemplateGenerationCommand { globTree(tree, glob) { debug('Checking for templates that include the glob pattern: %s', glob); const filterFunc = minimatch.filter(glob, { matchBase: true }); - return _.filter(tree, (f) => f.type === 'blob' && filterFunc(f.path)); + const res = _.filter(tree, (f) => f.type === 'blob' && filterFunc(f.path)); + debug('Number of results matching glob pattern "%s": %s', glob, res.length); + return res; } /** @@ -236,6 +241,7 @@ export class TemplateGenerationCommand { const loadMetadata = async (value) => { // TODO: this should have error handling to avoid a bad template crashing the CLI const data = JSON.parse((await this.readFile(value.path)).toString()); + debug('Successfully read metadata file: %s', value.path); return { name: data.title, value: { ...data, path: value.path }, @@ -299,13 +305,44 @@ export class TemplateGenerationCommand { [this.resourceTemplateName]: generateNameFromTitle(templateName), }); } catch (err) { - printError(err.message, this.options); + this.handleTemplatingError(templateName, f, err); } return undefined; }).join('
'); return generatedFiles; } + /** + * Error handler for templating specific scenarios - exits the program. + * + * @param {string} templateName Name of the template being copied + * @param {GitTreeWrapper} file File object that was being templated + * @param {Error} err Error that was thrown + */ + handleTemplatingError(templateName, file, err) { + debug('Template Name: %s', templateName); + debug('File details: %s', file); + debug('Error details: %s', err); + const message = `Failed to generate file "${file.path}" in template "${templateName}"!\n` + + `This is possibly the result of the ${this.resourceName} template having an invalid syntax.\n` + + `\nError: ${err.message}`; + printError(message, this.options); + } + + /** + * Returns whether a given file (path) in the template should be excluded from the template. + * Should return `true` if the file should be copied without modification. + * + * The default implementation allows for all files to be templated. + * + * @param {string} filepath + * @returns {boolean} true if the file should NOT be templated, false otherwise + */ + // eslint-disable-next-line no-unused-vars + filterByFileName(filepath) { + return false; + } + /** * Generates the selected template by templating its contents and writing its content to disk. * Returns the file tree with what files were created. @@ -328,8 +365,8 @@ export class TemplateGenerationCommand { template: template.template, }; let buf = await this.readFile(f.path); - /// Try not to template any non-text files. - if (isText(null, buf)) { + /// Try not to template any non-text files, and allow for a filter + if (isText(null, buf) && !this.filterByFileName(f.path)) { buf = Buffer.from(_.template(buf.toString(), { interpolate: /{{([\s\S]+?)}}/g })(templateVars)); } const relPath = f.path.slice(path.dirname(template.template.path).length); @@ -340,7 +377,7 @@ export class TemplateGenerationCommand { _.set(treeObj, sourcePath.split('/'), null); } } catch (err) { - printError(err.message, this.options); + this.handleTemplatingError(template.name, f, err); } })); return treeObj; From 295232db67f72db7e4956cdaffd93710384f9de7 Mon Sep 17 00:00:00 2001 From: Luis Aguirre Date: Tue, 2 Apr 2024 11:49:58 -0500 Subject: [PATCH 14/20] Rename method name and update coumment --- src/commands/pipelines.js | 3 +-- src/commands/workspaces/generate.js | 23 +++++++++++++---------- 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/src/commands/pipelines.js b/src/commands/pipelines.js index 860ac4e4..ff36c4c7 100644 --- a/src/commands/pipelines.js +++ b/src/commands/pipelines.js @@ -261,8 +261,7 @@ export const PipelineGenerateCommand = class extends TemplateGenerationCommand { super(program, 'Pipeline', 'pipelines', 'pipelinename', 'pipelineTemplateConfig'); } - // TODO: Need a better way to handle this filtering - filterByFileName(filepath) { + shouldTemplateFile(filepath) { // NOTE: It is common for dbt's templating (jinja2?) to collide with Lodash's templating syntax. // Current workaround is to exclude DBT & SQL files from templating, only downside is that those // can't use the {{pipelinename}} variable. diff --git a/src/commands/workspaces/generate.js b/src/commands/workspaces/generate.js index e41ec21d..ebdc2e9e 100644 --- a/src/commands/workspaces/generate.js +++ b/src/commands/workspaces/generate.js @@ -69,8 +69,8 @@ export class TemplateGenerationCommand { if (typeof this.configureSubcommand !== 'function') { throw new TypeError('Cannot construct instance of TemplateGenerationCommand without overriding async method "configureSubcommand()"!'); } - if (typeof this.filterByFileName !== 'function') { - throw new TypeError('Cannot construct instance of TemplateGenerationCommand without overriding async method "filterByFileName(string) -> bool"!'); + if (typeof this.shouldTemplateFile !== 'function') { + throw new TypeError('Cannot construct instance of TemplateGenerationCommand without overriding async method "shouldTemplateFile(string) -> bool"!'); } // Set options controlling direct behavior this.program = program; @@ -89,6 +89,9 @@ export class TemplateGenerationCommand { * @param {option} options CLI runtime options */ async init(name, destination, options) { + // NOTE: The code originally assigned below properties across multiple methods, which + // made keeping track of properties difficult. These assignments have been grouped + // together in a single method for simplicity. this.options = options; this.name = name; this.destination = destination; @@ -330,17 +333,17 @@ export class TemplateGenerationCommand { } /** - * Returns whether a given file (path) in the template should be excluded from the template. - * Should return `true` if the file should be copied without modification. + * Returns whether a given file (path) should have templating applied to it. + * Should return `false` if the file should be copied without modification. * - * The default implementation allows for all files to be templated. + * The default implementation always returns `true` - allowing all files to be templated. * * @param {string} filepath - * @returns {boolean} true if the file should NOT be templated, false otherwise + * @returns {boolean} true if the file should be templated, false otherwise. */ // eslint-disable-next-line no-unused-vars - filterByFileName(filepath) { - return false; + shouldTemplateFile(filepath) { + return true; } /** @@ -365,8 +368,8 @@ export class TemplateGenerationCommand { template: template.template, }; let buf = await this.readFile(f.path); - /// Try not to template any non-text files, and allow for a filter - if (isText(null, buf) && !this.filterByFileName(f.path)) { + /// Try not to template any non-text files and allow filering based on file names + if (isText(null, buf) && this.shouldTemplateFile(f.path)) { buf = Buffer.from(_.template(buf.toString(), { interpolate: /{{([\s\S]+?)}}/g })(templateVars)); } const relPath = f.path.slice(path.dirname(template.template.path).length); From 08a3c28f4798f42ad0eda7445d7a5ed79e0c7b1a Mon Sep 17 00:00:00 2001 From: Luis Aguirre Date: Tue, 2 Apr 2024 13:31:02 -0500 Subject: [PATCH 15/20] Remove usage of printError() in new CLI commands and wrap with callCommand() --- bin/cortex-pipelines.js | 16 ++-------------- 1 file changed, 2 insertions(+), 14 deletions(-) diff --git a/bin/cortex-pipelines.js b/bin/cortex-pipelines.js index 5befd596..33b8ae16 100755 --- a/bin/cortex-pipelines.js +++ b/bin/cortex-pipelines.js @@ -118,26 +118,14 @@ export function create() { .command('configure') .option('--refresh', 'Refresh the Github access token') .description('Configure the Cortex Template system for generating Pipeline templates') - .action((options) => { - try { - return new PipelineTemplateConfigureCommand(pipelines).execute(options); - } catch (err) { - return printError(err.message); - } - }); + .action(callCommand((options) => new PipelineTemplateConfigureCommand(pipelines).execute(options))); // Generate a Pipeline template pipelines.command('generate [pipelineName] [destination]') .option('--notree [boolean]', 'Do not dispaly generated file tree', 'false') .option('--template ', 'Name of the template to use') .description('Generates a folder based on a Pipeline template from the template repository') - .action((pipelineName, destination, options) => { - try { - return new PipelineGenerateCommand(pipelines).execute(pipelineName, destination, options); - } catch (err) { - return printError(err.message); - } - }); + .action(callCommand((pipelineName, destination, options) => new PipelineGenerateCommand(pipelines).execute(pipelineName, destination, options))); return pipelines; } From 423be0c0f8fb5a94a4dedba31445eafb6ccc26a2 Mon Sep 17 00:00:00 2001 From: Luis Aguirre Date: Fri, 5 Apr 2024 14:15:24 -0500 Subject: [PATCH 16/20] Add color & profile flags to Pipeline generate & configure CLI subcommands. Add validation for Pipeline Names - must match pre-configure regex --- bin/cortex-pipelines.js | 11 +++++++++-- bin/cortex-workspaces.js | 3 ++- src/commands/pipelines.js | 22 ++++++++++++++++++---- 3 files changed, 29 insertions(+), 7 deletions(-) diff --git a/bin/cortex-pipelines.js b/bin/cortex-pipelines.js index 33b8ae16..6acd7c42 100755 --- a/bin/cortex-pipelines.js +++ b/bin/cortex-pipelines.js @@ -113,19 +113,26 @@ export function create() { })); - // Configure Pipeline Template Github Repository + // Configure Pipeline Template Github Repository for the current CLI Profile (does not support --profile) pipelines .command('configure') + .option('--color [boolean]', 'Turn on/off colors', 'true') .option('--refresh', 'Refresh the Github access token') .description('Configure the Cortex Template system for generating Pipeline templates') .action(callCommand((options) => new PipelineTemplateConfigureCommand(pipelines).execute(options))); // Generate a Pipeline template pipelines.command('generate [pipelineName] [destination]') + .option('--profile ', 'The profile to use') + .option('--color [boolean]', 'Turn on/off colors', 'true') .option('--notree [boolean]', 'Do not dispaly generated file tree', 'false') .option('--template ', 'Name of the template to use') .description('Generates a folder based on a Pipeline template from the template repository') - .action(callCommand((pipelineName, destination, options) => new PipelineGenerateCommand(pipelines).execute(pipelineName, destination, options))); + .action(callCommand((pipelineName, destination, options) => { + checkForEmptyArgs({ pipelineName, destination }); + PipelineGenerateCommand.validatePipelineName(pipelineName, options); + return new PipelineGenerateCommand(pipelines).execute(pipelineName, destination, options); + })); return pipelines; } diff --git a/bin/cortex-workspaces.js b/bin/cortex-workspaces.js index 1ebc6817..ebba1fac 100755 --- a/bin/cortex-workspaces.js +++ b/bin/cortex-workspaces.js @@ -11,9 +11,10 @@ import { export function create() { const program = new Command(); - program.name('cortex workspaces'); program.description('Scaffolding Cortex Components'); + + // Configure Template Github Repository for the current CLI Profile (does not support --profile) program .command('configure') .option('--refresh', 'Refresh the Github access token') diff --git a/src/commands/pipelines.js b/src/commands/pipelines.js index ff36c4c7..9eecc6b0 100644 --- a/src/commands/pipelines.js +++ b/src/commands/pipelines.js @@ -263,16 +263,30 @@ export const PipelineGenerateCommand = class extends TemplateGenerationCommand { shouldTemplateFile(filepath) { // NOTE: It is common for dbt's templating (jinja2?) to collide with Lodash's templating syntax. - // Current workaround is to exclude DBT & SQL files from templating, only downside is that those - // can't use the {{pipelinename}} variable. + // Current workaround is to exclude specific DBT & SQL files from templating const fileName = path.posix.basename(filepath); - if (path.extname(fileName) === '.sql' || filepath.includes('dbt_packages') || filepath.includes('/dbt/')) { + if (fileName === 'dbt_projects.yml' || fileName === 'dbt_project.yaml') { + // Allow templating in the main dbt project file return true; } - return false; + if (path.extname(fileName) === '.sql' || filepath.includes('/dbt_packages/') || filepath.includes('/dbt')) { + return false; + } + return true; } async configureSubcommand() { await (new PipelineTemplateConfigureCommand(this.program)).execute({ refresh: true }); } + + static validatePipelineName(pipelineName, options) { + // Pipelines must have a name that is only lowercase characters, numbers, and underscore. + // Need to validate this before hand, otherwise the generated template will be unusable. + const nameRegex = /^[a-z](_?[a-z0-9]+)*$/g; // Taken from 'sensa-data-pipelines' package + if (!pipelineName || !nameRegex.test(pipelineName)) { + // Print error & exit + printError(`Cannot generate Pipeline with name "${pipelineName}". Pipeline names must conform to the regex: ` + + `${nameRegex} (all lowercase characters, with numbers or underscores)`, options); + } + } }; From 3ceeb24e2e5959e629429c042a09d9ecb1df1641 Mon Sep 17 00:00:00 2001 From: Luis Aguirre Date: Fri, 5 Apr 2024 16:58:30 -0500 Subject: [PATCH 17/20] Fix bug when checking for Empty strings - value might be null/undefined --- src/commands/utils.js | 6 +++--- src/compatibility.js | 1 + 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/commands/utils.js b/src/commands/utils.js index 09bc0604..afe0b8e4 100644 --- a/src/commands/utils.js +++ b/src/commands/utils.js @@ -529,9 +529,9 @@ export function printErrorDetails(response, options, exit = true) { } function checkForEmptyString(key, value) { - if (!value.trim()) { - printError(`error: <${key}> cannot be empty.`); - process.exit(1); // Exit with an error code + if (!(value?.trim())) { + printError(`error: <${key}> cannot be empty.`); + process.exit(1); // Exit with an error code } } diff --git a/src/compatibility.js b/src/compatibility.js index cbd6a0a6..9fc55314 100644 --- a/src/compatibility.js +++ b/src/compatibility.js @@ -138,6 +138,7 @@ export function callCommand(fn) { } return fn(...args); } catch (err) { + debug('Error: %s\n\nStack: %s', err, err?.stack); handleError(err); } }; From e8842489c04364e9a9554982245fc5ec34eb76e7 Mon Sep 17 00:00:00 2001 From: Luis Aguirre Date: Fri, 5 Apr 2024 18:24:04 -0500 Subject: [PATCH 18/20] Fix typo in --notree option description --- bin/cortex-pipelines.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/cortex-pipelines.js b/bin/cortex-pipelines.js index 6acd7c42..38efcce4 100755 --- a/bin/cortex-pipelines.js +++ b/bin/cortex-pipelines.js @@ -125,7 +125,7 @@ export function create() { pipelines.command('generate [pipelineName] [destination]') .option('--profile ', 'The profile to use') .option('--color [boolean]', 'Turn on/off colors', 'true') - .option('--notree [boolean]', 'Do not dispaly generated file tree', 'false') + .option('--notree [boolean]', 'Do not display generated file tree', 'false') .option('--template ', 'Name of the template to use') .description('Generates a folder based on a Pipeline template from the template repository') .action(callCommand((pipelineName, destination, options) => { From 4a87188df794daf11f953b4197ff28fd84bbf225 Mon Sep 17 00:00:00 2001 From: Luis Aguirre Date: Mon, 8 Apr 2024 15:05:48 -0500 Subject: [PATCH 19/20] Fix typo in dbt_project name --- src/commands/pipelines.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/commands/pipelines.js b/src/commands/pipelines.js index 9eecc6b0..2b53d7b9 100644 --- a/src/commands/pipelines.js +++ b/src/commands/pipelines.js @@ -265,7 +265,7 @@ export const PipelineGenerateCommand = class extends TemplateGenerationCommand { // NOTE: It is common for dbt's templating (jinja2?) to collide with Lodash's templating syntax. // Current workaround is to exclude specific DBT & SQL files from templating const fileName = path.posix.basename(filepath); - if (fileName === 'dbt_projects.yml' || fileName === 'dbt_project.yaml') { + if (fileName === 'dbt_project.yml' || fileName === 'dbt_project.yaml') { // Allow templating in the main dbt project file return true; } From 91ab6072fb3dfd8b5743f70a8aeb5957d5b14210 Mon Sep 17 00:00:00 2001 From: Luis Aguirre Date: Thu, 11 Apr 2024 15:57:10 -0500 Subject: [PATCH 20/20] Fix backwards compatibility issue from resourceType change for oskills --- src/commands/workspaces/generate.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/commands/workspaces/generate.js b/src/commands/workspaces/generate.js index ebdc2e9e..b7950518 100644 --- a/src/commands/workspaces/generate.js +++ b/src/commands/workspaces/generate.js @@ -250,8 +250,9 @@ export class TemplateGenerationCommand { value: { ...data, path: value.path }, }; }; + // default resourceType to Skill for backwards compatibility + const filterByResourceType = (data) => (data.value?.resourceType || 'Skill') === this.resourceName; const filterByEnabled = (data) => data.value?.enabled; - const filterByResourceType = (data) => (data.value?.resourceType) === this.resourceName; // Find possible choices for the template selection const registryUrl = await this.getRegistry();