diff --git a/README.md b/README.md index 11420d83..50479234 100644 --- a/README.md +++ b/README.md @@ -134,7 +134,7 @@ Executables are created for the following platforms: ## Subcommands & Compatibility Checks -The following sequene diagram shows an example running a command with Cortex CLI - namely which HTTP requests. ([This diagram can be viewered here](https://mermaid.live/view#pako:eNp9VE1z2jAQ_Ss7nh6aAeJLT54OHcZAQ4c4TEx64iLLa6IiS6ok0zCZ_PdKlg0k0Ookad--_XgrvUZUlhglkcHfDQqKU0a2mtQbAW6RxkrR1AXqcJ5QKzVMcR-OhXyBdLmAVNY1EaUJt34poi2jTBFhIQVigEpt8eU6IG-KEwQUU8iZwI4NRRk25x5zxtEcjMX60pZ7rhyFITBZLS7tmbdnq_tgcbWMxuNBmlyEB86M3YgAo5o5AsJhKUkJKy0rl0Ewpd7_lFACj7PJFD7dPdzP4ttAGlMpKrYN-BO0C5zyxp00PD0uh576F1I7BLT0to_ehsgTuFuvV_B9toa4IoVmNN5_iZmoZEDlHd-P_CGDzznqvSNdsxoht6RWQ5gjsY1GmHOyNTc99ah1WrKatUrQTksoiMESpICqc6u825ko3YhwCwtz5um2Svmyy28nuX0FDpJAiRy3xCJYCebo07Fyg-9dBk6eBGZaS53Ak9gJ-Uf003aRSSYdrfRFu0jDzAnRtGBFLCsYZ_YA6TPS3YWmVzFtR5vC53298fTcKyZKcUdmmRQmDqqPKGdnyrTlB20e3Vtj2nX3J2rjPG76lELAQXYWsWti9oFisieMk4Jj-wA7nl7UfzYlD01xz9ihL2fbDTys3j-__7ZAhWE18dduN4775zPSqKRh7rtgR6oPJRwjnbKOhlGNuiasdD_Sq7_eRPYZa9xEiduWRO820Ua8OZz_mvKDoFFidYPDqFGlG6vu9wqXb38B-i2SbA)). +The following sequence diagram shows an example running a command with Cortex CLI - namely which HTTP requests. ([This diagram can be viewered here](https://mermaid.live/view#pako:eNp9VE1z2jAQ_Ss7nh6aAeJLT54OHcZAQ4c4TEx64iLLa6IiS6ok0zCZ_PdKlg0k0Ookad--_XgrvUZUlhglkcHfDQqKU0a2mtQbAW6RxkrR1AXqcJ5QKzVMcR-OhXyBdLmAVNY1EaUJt34poi2jTBFhIQVigEpt8eU6IG-KEwQUU8iZwI4NRRk25x5zxtEcjMX60pZ7rhyFITBZLS7tmbdnq_tgcbWMxuNBmlyEB86M3YgAo5o5AsJhKUkJKy0rl0Ewpd7_lFACj7PJFD7dPdzP4ttAGlMpKrYN-BO0C5zyxp00PD0uh576F1I7BLT0to_ehsgTuFuvV_B9toa4IoVmNN5_iZmoZEDlHd-P_CGDzznqvSNdsxoht6RWQ5gjsY1GmHOyNTc99ah1WrKatUrQTksoiMESpICqc6u825ko3YhwCwtz5um2Svmyy28nuX0FDpJAiRy3xCJYCebo07Fyg-9dBk6eBGZaS53Ak9gJ-Uf003aRSSYdrfRFu0jDzAnRtGBFLCsYZ_YA6TPS3YWmVzFtR5vC53298fTcKyZKcUdmmRQmDqqPKGdnyrTlB20e3Vtj2nX3J2rjPG76lELAQXYWsWti9oFisieMk4Jj-wA7nl7UfzYlD01xz9ihL2fbDTys3j-__7ZAhWE18dduN4775zPSqKRh7rtgR6oPJRwjnbKOhlGNuiasdD_Sq7_eRPYZa9xEiduWRO820Ua8OZz_mvKDoFFidYPDqFGlG6vu9wqXb38B-i2SbA)). The Cortex CLI will limit the set of available subcommands, to those which are supported by the Cortex cluster that it is communicating with. diff --git a/bin/cortex-pipelines.js b/bin/cortex-pipelines.js index 10eec48e..38efcce4 100755 --- a/bin/cortex-pipelines.js +++ b/bin/cortex-pipelines.js @@ -7,6 +7,8 @@ import { RunPipelineCommand, DescribePipelineRunCommand, ListPipelineRunsCommand, + PipelineTemplateConfigureCommand, + PipelineGenerateCommand, } from '../src/commands/pipelines.js'; import { callCommand } from '../src/compatibility.js'; import { @@ -110,6 +112,28 @@ export function create() { return new ListPipelineRunsCommand(pipelines).execute(pipelineName, gitRepoName, options); })); + + // Configure Pipeline Template Github Repository for the current CLI Profile (does not support --profile) + pipelines + .command('configure') + .option('--color [boolean]', 'Turn on/off colors', 'true') + .option('--refresh', 'Refresh the Github access token') + .description('Configure the Cortex Template system for generating Pipeline templates') + .action(callCommand((options) => new PipelineTemplateConfigureCommand(pipelines).execute(options))); + + // Generate a Pipeline template + pipelines.command('generate [pipelineName] [destination]') + .option('--profile ', 'The profile to use') + .option('--color [boolean]', 'Turn on/off colors', 'true') + .option('--notree [boolean]', 'Do not display generated file tree', 'false') + .option('--template ', 'Name of the template to use') + .description('Generates a folder based on a Pipeline template from the template repository') + .action(callCommand((pipelineName, destination, options) => { + checkForEmptyArgs({ pipelineName, destination }); + PipelineGenerateCommand.validatePipelineName(pipelineName, options); + return new PipelineGenerateCommand(pipelines).execute(pipelineName, destination, options); + })); + return pipelines; } diff --git a/bin/cortex-workspaces.js b/bin/cortex-workspaces.js index dcdd0310..ebba1fac 100755 --- a/bin/cortex-workspaces.js +++ b/bin/cortex-workspaces.js @@ -1,8 +1,8 @@ import { Command } from 'commander'; import process from 'node:process'; import esMain from 'es-main'; -import WorkspaceConfigureCommand from '../src/commands/workspaces/configure.js'; -import WorkspaceGenerateCommand from '../src/commands/workspaces/generate.js'; +import { WorkspaceConfigureCommand } from '../src/commands/workspaces/configure.js'; +import { WorkspaceGenerateCommand } from '../src/commands/workspaces/generate.js'; import WorkspaceBuildCommand from '../src/commands/workspaces/build.js'; import WorkspacePublishCommand from '../src/commands/workspaces/publish.js'; import { @@ -11,9 +11,10 @@ import { export function create() { const program = new Command(); - program.name('cortex workspaces'); program.description('Scaffolding Cortex Components'); + + // Configure Template Github Repository for the current CLI Profile (does not support --profile) program .command('configure') .option('--refresh', 'Refresh the Github access token') diff --git a/docs/workspaces.md b/docs/workspaces.md new file mode 100644 index 00000000..344a6942 --- /dev/null +++ b/docs/workspaces.md @@ -0,0 +1,237 @@ +# Workspaces + +## Overview + +Note that `workspaces` CLI commands are narrowly focused on generating Skill templates. Ideally `workspaces` would encompass Pipeline as well, but the initial implementation for Pipelines uses its CLI commands (similar to `workspaces`). + +Both commands only support Git Repositories hosted on GitHub! + +### Templates + +The primary repository with templates can be found at: + +* A template is any folder within the git Repository that contains a `metadata.json` file. + + Expected structure for `metadata.json`: + + ```json + { + "name": "", + "title": "", + "description": "", + "tags": ["list", "of", "tags", "applied", "to", "template"], + "enabled": true, + "resourceType": "Skill" // e.g. "Pipeline" + } + ``` + +* Templates can be nested in any subfolder in the repository +* The templating process works via the [loadsh template](https://docs-lodash.com/v4/template/) util. Refer to that documentation for how variables are templated. + +### Additional Reference + +* [Example Cortex Development Workflow](https://drive.google.com/file/d/1tPyuqtNFz9JFtJuQE6HRxou_SLKIK8fO/view?usp=drive_link) +* [Relation to Skill Building](https://docs.google.com/presentation/d/1k4vJ7d5oGbvFaUezl5dBXHtKc-CZK8IAGlvIq-RpJrk/edit#slide=id.g12c69d77b32_0_42) (i.e. migration from traditional Skill Building) + +## Workspaces Configure + +The configuration process implements the [Github OAuth Device Flow](https://docs.github.com/en/apps/oauth-apps/building-oauth-apps/authorizing-oauth-apps#device-flow). The reason for implementing this flow is primarily to avoid rate limiting issues from the GitHub rest API, see: + +* [Unauthenticated User Rate Limiting](https://docs.github.com/en/rest/using-the-rest-api/rate-limits-for-the-rest-api?apiVersion=2022-11-28#primary-rate-limit-for-unauthenticated-users) (~60 requests per hour) +* [Authenticated User Rate Limiting](https://docs.github.com/en/rest/using-the-rest-api/rate-limits-for-the-rest-api?apiVersion=2022-11-28#primary-rate-limit-for-authenticated-users) (~5000 requests per hour) + +The GitHub authentication does NOT allow users to access private repositories. Using a private repository would require [registering the Cortex CLI App in your developer settings](https://github.com/orgs/community/discussions/48102). + +```mermaid +%% Title: Cortex Workspaces configure CLI command +sequenceDiagram + autonumber + title Workspaces Configure CLI subcommand + Actor D as Dev + participant C as Cortex CLI + participant F as Filesystem + + %% Mac: KeyChain + %% Linux: Secret Service API/libsecret + %% Windows: Credential Vault + participant K as System Keychain
(Optional) + + participant B as Browser + participant G as GitHub + + D->>+C: Configure CLI + Note over D,C: cortex workspaces configure --refresh --no-timeout + + C->>+F: Read Config File
fetch existing config + C->>+D: Prompt user for Repository/Branch + C->>+D: User responds with answers + + %% TODO: Check if token is valid + %% If it's invalid and not being refreshed - exit with error + + C->>+B: Browser Opens at GitHub + D->>+B: Enter code & authorize Cortex CLI + loop Fixed Interval (e.g. 5 seconds) + C->>+G: Fetch GitHub Device Code + G->>+C: JSON (Potential Device Code) + Note over C,G: HHTP POST https://github.com/login/device/code + + critical Check Access Token + option Access Token returned + alt Keychain service available + C->>+K: Store token in Keychain + else Local File fallback + C->>+F: Store token as file in Config folder + end + C->>+D: Configuration Successful + option Authorization Pending + C->>+C: Update Expiration Time + option Slow Down + C->>+C: Increment Poll Interval + option Expired Token + C->>+D: Exit with error - re-configure and try again + option Incorrect Device Code + C->>+D: Exit with error - incorrect device code entered + option Access Denied + C->>+D: Exit with error - Access denied by user + option Unexpected Error + C->>+D: Exit with error + end + end +``` + + + +## Workspaces Generate + +The following diagram shows the sequence of steps taken when generating a template. + +```mermaid +sequenceDiagram + autonumber + title Workspaces Generate CLI subcommand + Actor D as Dev + participant C as Cortex CLI + participant F as Filesystem + + %% Mac: KeyChain + %% Linux: Secret Service API/libsecret + %% Windows: Credential Vault + participant K as System Keychain
(Optional) + participant G as GitHub + + D->>+C: Generate Pipeline + Note over D,C: cortex workspaces generate + + critical Github Token Validation + C->>+F: Read the user config File + alt Keychain service available + C->>+K: Load token from Keychain + else Local File fallback + C->>+F: Load token from cached file in Config folder + end + %% Valdiate the Token + C->>+G: Validate Token with Github API + Note over C,G: HTTP GET https://api.github.com/user (with Token) + + option Token is Invalid + C->>+C: Force user to configure template repository (recreate Token) + option Token is Valid + C->>+C: Continue (No-Op) + end + + opt Fail early if destination already exists + C->>+F: Check if the destination exists + C->>+D: Report Error to user + end + + critical Fetch Git Tree for the Repository/Branch + C->>+G: Fetch the HEAD of the configured repository/branch + G->>+C: git SHA (JSON) + Note over C,G: HTTP GET https://api.github.com/repos//branches/ + alt Repository & Branch exist + C->>+G: Fetch Git Tree(s) from the Repo + G->>+C: Git Tree (JSON)/fetch + Note over C,G: HTTP GET https://api.github.com/repos//git/trees/?recursive=true + end + end + + critical Select Template + Note right of C: Templates are identifed based
on presence of `metadata.json` + C->>+C: Use glob to find potential templates in Git Tree + loop For each potential template + C->>+G: Read the metadata.json file for the Template + G->>+C: JSON + Note over C,G: HTTP GET https://api.github.com/repos//contents/?ref= + end + + C->>+D: Prompt User for template selection + D->>+C: Answers (Template Choice, Template Name, etc.) + alt No Templates found + C->>+D: Display Error + end + alt Selected Template does not exist + C->>+D: Display Error + end + end + + critical Generate Template + C->>+C: Use glob to identify files corresponding to template in Git Tree + + opt Fail early if destination already exists + C->>+F: Check if the destination exists + C->>+D: Report Error to user + end + + C->>+C: Compute generated files
with templated values + loop For each templated file + C->>+G: Fetch file contents + G->>+C: File Contents (stream) + Note over C,G: HTTP GET https://api.github.com/repos//contents/?ref= + C->>+C: Apply templating + C->>+F: Write templated file to Filesystem + end + end + + opt Display generated file tree to user + C->>+D: Print the generated file tree + end +``` + +## Relation to Pipelines + +The Cortex CLI offers similar capabilities for generating Pipelines template, similar to Workspaces, however, the both are distinct features (subcommands) in the CLI. + +### Pipelines Configure + +The process is simlilar to that of [workspaces configure](#workspaces-configure), but: + +* A separate section of the Config file is used to store the Repository configuration - i.e. The Pipeline Template repository can be different + +```bash +$ cortex pipelines configure --refresh --no-timeout +Configuring workspaces for profile qa-aks +? Template Repository URL: CognitiveScale/cortex-code-templates +? Template Repository Branch: FAB-6046-pipeline-generate +Opening browser at https://github.com/login/device +Please enter the following code to authorize the Cortex CLI: FB05-0378 ( Expires in 14 minutes and 58 seconds ) - CTRL-C to abort +Github token configuration successful. +``` + +### Pipelines Generate + +The process is simlilar to that of [workspaces generate](#workspaces-generate), but only Pipeline templates are available. + +```bash +$ cortex pipelines generate +``` + +## Workspaces vs Pipelines + +The Cortex CLI offers similar capabilities for generating Pipelines template, similar to Workspaces, however, the both are distinct features (subcommands) in the CLI. diff --git a/src/commands/pipelines.js b/src/commands/pipelines.js index dbe39670..2b53d7b9 100644 --- a/src/commands/pipelines.js +++ b/src/commands/pipelines.js @@ -1,6 +1,7 @@ import debugSetup from 'debug'; import fs from 'fs'; import dayjs from 'dayjs'; +import path from 'path'; import _ from 'lodash'; import relativeTime from 'dayjs/plugin/relativeTime.js'; import { loadProfile } from '../config.js'; @@ -17,12 +18,14 @@ import { printTable, printWarning, handleError, } from './utils.js'; +import { TemplateConfigureCommand } from './workspaces/configure.js'; +import { TemplateGenerationCommand } from './workspaces/generate.js'; + const debug = debugSetup('cortex:cli'); dayjs.extend(relativeTime); - export const ListPipelineCommand = class { constructor(program) { this.program = program; @@ -241,3 +244,49 @@ export const ListPipelineRunsCommand = class { } } }; + +// NOTE: Easiest way to piggy-back of the existing functionality from Workspaces is to +// directly use the same logic via inheritance. +// +// The constructor assigns a different configKey to avoid collision from sharing the +// same property in the config file. +export const PipelineTemplateConfigureCommand = class extends TemplateConfigureCommand { + constructor(program) { + super(program, 'pipelineTemplateConfig', 'Pipelines'); + } +}; + +export const PipelineGenerateCommand = class extends TemplateGenerationCommand { + constructor(program) { + super(program, 'Pipeline', 'pipelines', 'pipelinename', 'pipelineTemplateConfig'); + } + + shouldTemplateFile(filepath) { + // NOTE: It is common for dbt's templating (jinja2?) to collide with Lodash's templating syntax. + // Current workaround is to exclude specific DBT & SQL files from templating + const fileName = path.posix.basename(filepath); + if (fileName === 'dbt_project.yml' || fileName === 'dbt_project.yaml') { + // Allow templating in the main dbt project file + return true; + } + if (path.extname(fileName) === '.sql' || filepath.includes('/dbt_packages/') || filepath.includes('/dbt')) { + return false; + } + return true; + } + + async configureSubcommand() { + await (new PipelineTemplateConfigureCommand(this.program)).execute({ refresh: true }); + } + + static validatePipelineName(pipelineName, options) { + // Pipelines must have a name that is only lowercase characters, numbers, and underscore. + // Need to validate this before hand, otherwise the generated template will be unusable. + const nameRegex = /^[a-z](_?[a-z0-9]+)*$/g; // Taken from 'sensa-data-pipelines' package + if (!pipelineName || !nameRegex.test(pipelineName)) { + // Print error & exit + printError(`Cannot generate Pipeline with name "${pipelineName}". Pipeline names must conform to the regex: ` + + `${nameRegex} (all lowercase characters, with numbers or underscores)`, options); + } + } +}; diff --git a/src/commands/utils.js b/src/commands/utils.js index 09bc0604..afe0b8e4 100644 --- a/src/commands/utils.js +++ b/src/commands/utils.js @@ -529,9 +529,9 @@ export function printErrorDetails(response, options, exit = true) { } function checkForEmptyString(key, value) { - if (!value.trim()) { - printError(`error: <${key}> cannot be empty.`); - process.exit(1); // Exit with an error code + if (!(value?.trim())) { + printError(`error: <${key}> cannot be empty.`); + process.exit(1); // Exit with an error code } } diff --git a/src/commands/workspaces/configure.js b/src/commands/workspaces/configure.js index 3ddb4b1f..16bf574d 100644 --- a/src/commands/workspaces/configure.js +++ b/src/commands/workspaces/configure.js @@ -17,29 +17,44 @@ const DEFAULT_TEMPLATE_BRANCH = 'main'; const GITHUB_DEVICECODE_REQUEST_URL = 'https://github.com/login/device/code'; const GITHUB_DEVICECODE_RESPONSE_URL = 'https://github.com/login/oauth/access_token'; -export default class WorkspaceConfigureCommand { - constructor(program) { +export class TemplateConfigureCommand { + /** + * Creates a Command object that prompts the user to configures a remote Github Repository & Branch + * as the source of for templates. + * + * @param {object} program Commander progrma object + * @param {string} configKey Key in user Config file to store the configuration settings under + * @param {string} context String context for what is being configured - e.g. 'Pipelines', 'Workspaces' + */ + constructor(program, configKey, context) { + // Adhoc way of implemeting an abstract class + if (new.target === TemplateConfigureCommand) { + throw new TypeError('Cannot construct TemplateConfigureCommand instances directly!'); + } this.program = program; + this.configKey = configKey; + this.context = context; } async execute(opts) { this.options = opts; try { const config = readConfig(); + const { configKey } = this; const currentProfile = config.profiles[config.currentProfile]; - console.log(`Configuring workspaces for profile ${chalk.green(config.currentProfile)}`); + console.log(`Configuring ${this.context} for profile ${chalk.green(config.currentProfile)}`); const answers = await inquirer.prompt([ { type: 'input', name: 'repo', message: 'Template Repository URL: ', - default: _.get(currentProfile, 'templateConfig.repo', DEFAULT_TEMPLATE_REPO), + default: _.get(currentProfile, `${configKey}.repo`, DEFAULT_TEMPLATE_REPO), }, { type: 'input', name: 'branch', message: 'Template Repository Branch:', - default: _.get(currentProfile, 'templateConfig.branch', DEFAULT_TEMPLATE_BRANCH), + default: _.get(currentProfile, `${configKey}.branch`, DEFAULT_TEMPLATE_BRANCH), }, ]); const githubToken = await validateToken(); @@ -84,7 +99,7 @@ export default class WorkspaceConfigureCommand { if (accessToken.access_token) { clearTimeout(pollTimer); persistToken(accessToken); - config.profiles[config.currentProfile].templateConfig = answers; + config.profiles[config.currentProfile][configKey] = answers; config.save(); printSuccess('\x1b[0G\x1b[2KGithub token configuration successful.', options); resolve(); @@ -125,7 +140,7 @@ export default class WorkspaceConfigureCommand { printError('\x1b[2KDevice Code request failed. Please try again.', this.options); } } else { - config.profiles[config.currentProfile].templateConfig = answers; + config.profiles[config.currentProfile][configKey] = answers; config.save(); } } catch (error) { @@ -133,3 +148,9 @@ export default class WorkspaceConfigureCommand { } } } + +export class WorkspaceConfigureCommand extends TemplateConfigureCommand { + constructor(program) { + super(program, 'templateConfig', 'workspaces'); + } +} diff --git a/src/commands/workspaces/generate.js b/src/commands/workspaces/generate.js index 79bc591a..b7950518 100644 --- a/src/commands/workspaces/generate.js +++ b/src/commands/workspaces/generate.js @@ -5,52 +5,191 @@ import { mkdir, writeFile } from 'fs/promises'; import { minimatch } from 'minimatch'; import inquirer from 'inquirer'; import ghGot from 'gh-got'; +import debugSetup from 'debug'; import { isText } from 'istextorbinary'; import treeify from 'treeify'; import { boolean } from 'boolean'; import _ from 'lodash'; -import WorkspaceConfigureCommand from './configure.js'; +import { WorkspaceConfigureCommand } from './configure.js'; import { readConfig, loadProfile } from '../../config.js'; import { printToTerminal, validateToken } from './workspace-utils.js'; import { printSuccess, printError, validateName, generateNameFromTitle, } from '../utils.js'; +const debug = debugSetup('cortex:cli'); + const METADATA_FILENAME = 'metadata.json'; -export default class WorkspaceGenerateCommand { - constructor(program) { + +// ANSI escape sequences for pretty printing messages during generation. +// Reference: https://gist.github.com/fnky/458719343aabd01cfb17a3a4f7296797#cursor-controls +const RESET_CURSOR_ANSI_ESCAPE = '\x1b[0G\x1b[2K'; +const CURSOR_NEXT_LINE_ANSI_ESCAPE = '\x1b[1E'; + +/** + * Class wrapping the raw results from fetching a Git Tree. + * + * (Primarily meant for type hinting). + */ +class GitTreeWrapper { + constructor({ + // eslint-disable-next-line no-shadow + path, mode, type, sha, size, url, + }) { + this.path = path; + this.mode = mode; + this.type = type; + this.sha = sha; + this.size = size; + this.url = url; + if (!this.path) { + throw TypeError('Raw Git Tree is missing attribute "path"'); + } + if (!this.type) { + throw TypeError('Raw Git Tree is missing attribute "type"'); + } + } +} + +export class TemplateGenerationCommand { + /** + * Creates a `BaseGenerateCommand`. + * + * @param {object} program Commander program object + * @param {string} resourceName Name of the resource being generated (e.g. Skill, Pipeline, Foo, Bar) + * @param {string} resourceFolderName Name of the folder in the repository that will be checked + * @param {string} resourceTemplateName Expected folder name for the template - will be wrapped in underscores (e.g. __skillname__, __pipelinename__) + * @param {string} configKey Key in Configuration to use for fetching the underlying Template Repository, Registry, etc. + */ + constructor(program, resourceName, resourceFolderName, resourceTemplateName, configKey) { + // Adhoc way of implemeting an abstract class that enforces a 'configureSubcommand()' method to be present + if (new.target === TemplateGenerationCommand) { + throw new TypeError('Cannot construct TemplateGenerationCommand instances directly!'); + } + if (typeof this.configureSubcommand !== 'function') { + throw new TypeError('Cannot construct instance of TemplateGenerationCommand without overriding async method "configureSubcommand()"!'); + } + if (typeof this.shouldTemplateFile !== 'function') { + throw new TypeError('Cannot construct instance of TemplateGenerationCommand without overriding async method "shouldTemplateFile(string) -> bool"!'); + } + // Set options controlling direct behavior this.program = program; + this.resourceName = resourceName; + this.resourceFolderName = resourceFolderName; + this.resourceTemplateName = resourceTemplateName; + this.configKey = configKey; } - async loadTemplateTree({ repo, branch }) { - printToTerminal('\x1b[0G\x1b[2KLoading templates...'); - this.gitRepo = repo; - this.branch = branch; + /** + * Initializes the `BaseGenerateCommand` by setting execution-time options and retrieving the template + * configuration from the config file. + * + * @param {string} name Desired name for the generated template + * @param {string} destination Path to where the template should be generated + * @param {option} options CLI runtime options + */ + async init(name, destination, options) { + // NOTE: The code originally assigned below properties across multiple methods, which + // made keeping track of properties difficult. These assignments have been grouped + // together in a single method for simplicity. + this.options = options; + this.name = name; + this.destination = destination; + this.config = readConfig(); + this.githubToken = await validateToken(); this.authorization = this.githubToken; - this.tree = await ghGot(`repos/${repo}/branches/${branch || 'main'}`, { + const templateConfig = this.getTemplateConfig(); + this.gitRepo = templateConfig.repo; + this.branch = templateConfig.branch; + debug('Initialized template configuration. Raw configuration: %s', JSON.stringify(templateConfig)); + debug('GitHub Token Set? %s', this.githubToken != null); + } + + /** + * Returns the configuration for the template repository/branch. + * + * @returns {object} The configuraiton for the template repository + */ + getTemplateConfig() { + return this.config.profiles[this.config.currentProfile][this.configKey]; + } + + /** + * Queries GitHub to get the Git Tree(s) from a repository. + * + * @param {string} repo Github repository identifier (e.g. `org/repo`) + * @param {string} sha Git SHA referencing the commit in the repository to retrieve + * @returns {object} object returned by the GitHub API + */ + fetchGitTree(repo, sha) { + const uri = `repos/${repo}/git/trees/${sha}?recursive=true`; + debug(`Querying Git Tree: ${uri}`); + return ghGot(uri, { headers: { authorization: this.authorization }, - }) - .then((resp) => resp.body) - .then((resp) => ghGot(`repos/${repo}/git/trees/${resp.commit.sha}?recursive=true`, { + }); + } + + /** + * Loads Git Tree(s) corresponding to the HEAD of configured repo/branch. + * + * @return {Array} array of Git tree objects + */ + async fetchGitTreeFromBranch() { + const ghUri = `repos/${this.gitRepo}/branches/${this.branch || 'main'}`; + debug('Loading templates from "%s" (branch = "%s"). URI: %s', this.gitRepo, this.branch, ghUri); + const options = { headers: { authorization: this.authorization }, - })) - .then((resp) => resp.body.tree) - .catch(() => []); - printToTerminal('\x1b[0G\x1b[2KTemplates loaded.\x1b[1E'); + }; + const tree = await ghGot(ghUri, options) + .then((resp) => resp.body) + .then((resp) => this.fetchGitTree(this.gitRepo, resp.commit.sha)) + .then((resp) => resp.body.tree) + .then((resp) => resp.map((t) => new GitTreeWrapper(t))) + .catch((e) => { + debug('Error encountered while trying to fetch the git tree for the repository. %s', e); + if (e?.response?.statusCode === 404) { + printError('Unable to retrieve templates. Repository or branch not found!', this.options); + } + }); + return tree; } - globTree(glob) { + /** + * Filters the Git Trees to only those that are blobs that match the glob pattern. + * + * @param {Array} tree array of git tree objects + * @param {string} glob pattern to match + * @returns {Array} instances matching the glob pattern + */ + globTree(tree, glob) { + debug('Checking for templates that include the glob pattern: %s', glob); const filterFunc = minimatch.filter(glob, { matchBase: true }); - return _.filter(this.tree, (f) => f.type === 'blob' && filterFunc(f.path)); + const res = _.filter(tree, (f) => f.type === 'blob' && filterFunc(f.path)); + debug('Number of results matching glob pattern "%s": %s', glob, res.length); + return res; } + /** + * Reads a file in the configured remote git repository. + * + * @param {string} filePath path to the file in the repository + * @returns `Buffer` A Buffer with the contents of the file + */ async readFile(filePath) { - const response = await ghGot(`repos/${this.gitRepo}/contents/${filePath}?ref=${this.branch}`, { + const uri = `repos/${this.gitRepo}/contents/${filePath}?ref=${this.branch}`; + debug('Reading file contents "%s" from git repository (uri = "%s")', filePath, uri); + const response = await ghGot(uri, { headers: { authorization: this.authorization }, }); + debug('Successfully read file contents "%s" from git repository', filePath); return Buffer.from(response.body.content, 'base64'); } + /** + * Return a URL the Docker registry associated with the current Profile. + * + * @returns string + */ async getRegistry() { const profile = await loadProfile(this.options.profile); const registryUrl = _.get(this, 'options.registry') @@ -59,25 +198,15 @@ export default class WorkspaceGenerateCommand { return registryUrl; } - async selectTemplate(templateName) { - const registryUrl = await this.getRegistry(); - const fileNames = this.globTree(METADATA_FILENAME); - const choices = await Promise.all(_.map(fileNames, async (value) => { - const data = JSON.parse((await this.readFile(value.path)).toString()); - return { - name: data.title, - value: { ...data, path: value.path }, - }; - })); - let template; - if (templateName) { - const templateChoice = _.find(choices, { name: templateName }); - if (templateChoice) { - template = templateChoice.value; - } else { - printError(`Template ${templateName} not found!`); - } - } + /** + * Prompts the user to select the desired template. + * + * @param {string} registryUrl URL to Docker registry + * @param {Array} choices + * @param {string | null | undefined} template + * @returns {object} object with user answers (`template`, `name`, `registry`) + */ + async promptUser(registryUrl, choices, template) { const answers = await inquirer.prompt([ { type: 'list', @@ -88,7 +217,7 @@ export default class WorkspaceGenerateCommand { { type: 'input', name: 'name', - message: 'Enter a name for the skill:', + message: `Enter a name for the ${this.resourceName}:`, validate: (answer) => { const validation = validateName(answer); return validation.status || validation.message; @@ -102,84 +231,228 @@ export default class WorkspaceGenerateCommand { return answers; } + /** + * Selects the template to clone by fetching the potential choices from the given + * Git Tree(s) and prompting the user for their desired template. + * + * @param {Array} tree Git Trees for the contents of repository + * @param {string | null | undefined} templateName Name of template + * @returns {object} object with user selections (`template`, `name`, `registry`) + */ + async selectTemplate(tree, templateName) { + // Utils for proessing the Metadata file + const loadMetadata = async (value) => { + // TODO: this should have error handling to avoid a bad template crashing the CLI + const data = JSON.parse((await this.readFile(value.path)).toString()); + debug('Successfully read metadata file: %s', value.path); + return { + name: data.title, + value: { ...data, path: value.path }, + }; + }; + // default resourceType to Skill for backwards compatibility + const filterByResourceType = (data) => (data.value?.resourceType || 'Skill') === this.resourceName; + const filterByEnabled = (data) => data.value?.enabled; + + // Find possible choices for the template selection + const registryUrl = await this.getRegistry(); + debug('Docker Registry URL: %s', registryUrl); + const fileNames = this.globTree(tree, METADATA_FILENAME); + const choices = (await Promise.all(_.map(fileNames, loadMetadata))) + .filter(filterByEnabled) + .filter(filterByResourceType); + debug(JSON.stringify(choices)); + debug('Potential Choices: %s', JSON.stringify(choices.map((c) => c.name))); + + // Prompt user for selected template (handle template specified up front) + let template; + if (templateName) { + const templateChoice = _.find(choices, { name: templateName }); + if (templateChoice) { + template = templateChoice.value; + } else { + printError(`Template ${templateName} not found!`); + } + } + const answers = await this.promptUser(registryUrl, choices, template); + return answers; + } + + /** + * Checks whether the path to where the Template will be generated already exists. If so, exists the program. + * + * @param {string} destinationPath Base directory + * @param {string} name Name of the resource to be generated + */ + checkDestinationExists(destinationPath, name) { + // Fail if destination already exists + if (fs.existsSync(path.join(destinationPath, this.resourceFolderName, name))) { + printError(`${this.resourceName} ${name} already exists!`, this.options); + } + } + + /** + * Applies templating over the filenames in the set of files to generate. + * + * @param {string} templateFolder Folder name for the template to be generated + * @param {Array} templateFiles Files that will be templated + * @param {string} templateName String name to apply to the template + * @returns {string} A string with all the filenames to generate separated by `
` + */ + applyTemplatingToFilenames(templateFolder, templateFiles, templateName) { + const generatedFiles = _.map(templateFiles, (f) => { + try { + const rootPathComponents = templateFolder.split('/'); + const destFile = _.drop(f.path.split('/'), rootPathComponents.length); + const targetPath = path.join(...destFile); + return _.template(targetPath, { interpolate: /__([\s\S]+?)__/g })({ + [this.resourceTemplateName]: generateNameFromTitle(templateName), + }); + } catch (err) { + this.handleTemplatingError(templateName, f, err); + } + return undefined; + }).join('
'); + return generatedFiles; + } + + /** + * Error handler for templating specific scenarios - exits the program. + * + * @param {string} templateName Name of the template being copied + * @param {GitTreeWrapper} file File object that was being templated + * @param {Error} err Error that was thrown + */ + handleTemplatingError(templateName, file, err) { + debug('Template Name: %s', templateName); + debug('File details: %s', file); + debug('Error details: %s', err); + const message = `Failed to generate file "${file.path}" in template "${templateName}"!\n` + + `This is possibly the result of the ${this.resourceName} template having an invalid syntax.\n` + + `\nError: ${err.message}`; + printError(message, this.options); + } + + /** + * Returns whether a given file (path) should have templating applied to it. + * Should return `false` if the file should be copied without modification. + * + * The default implementation always returns `true` - allowing all files to be templated. + * + * @param {string} filepath + * @returns {boolean} true if the file should be templated, false otherwise. + */ + // eslint-disable-next-line no-unused-vars + shouldTemplateFile(filepath) { + return true; + } + + /** + * Generates the selected template by templating its contents and writing its content to disk. + * Returns the file tree with what files were created. + * + * @param {string} destinationPath path to generate template at + * @param {string} generatedFiles String with all the filenames that will be created + * @param {object} template Object with `template`, `name`, and `registry` + * @param {Array} templateFiles Array of objects representing files to generate + * @returns {object} An object containing the file tree for all generated files + */ + async generateTemplatedFiles(destinationPath, generatedFiles, template, templateFiles) { + const treeObj = {}; + await Promise.all(_.map(templateFiles, async (f) => { + try { + const fileName = path.posix.basename(f.path); + if (fileName !== METADATA_FILENAME) { + const templateVars = { + [this.resourceTemplateName]: generateNameFromTitle(template.name), + generatedFiles, + template: template.template, + }; + let buf = await this.readFile(f.path); + /// Try not to template any non-text files and allow filering based on file names + if (isText(null, buf) && this.shouldTemplateFile(f.path)) { + buf = Buffer.from(_.template(buf.toString(), { interpolate: /{{([\s\S]+?)}}/g })(templateVars)); + } + const relPath = f.path.slice(path.dirname(template.template.path).length); + const sourcePath = _.template(relPath, { interpolate: /__([\s\S]+?)__/g })(templateVars); + const targetPath = path.join(destinationPath, sourcePath); + await mkdir(path.dirname(targetPath), { recursive: true }); + await writeFile(targetPath, buf); + _.set(treeObj, sourcePath.split('/'), null); + } + } catch (err) { + this.handleTemplatingError(template.name, f, err); + } + })); + return treeObj; + } + + /** + * Generates the specified template at the given destination. + * + * @param {Array { - try { - const rootPathComponents = templateFolder.split('/'); - const destFile = _.drop(f.path.split('/'), rootPathComponents.length); - const targetPath = path.join(...destFile); - return _.template(targetPath, { interpolate: /__([\s\S]+?)__/g })({ - skillname: generateNameFromTitle(template.name), - }); - } catch (err) { - printError(err.message, this.options); - } - return undefined; - }).join('
'); - console.log(''); - await Promise.all(_.map(templateFiles, async (f) => { - try { - const fileName = path.posix.basename(f.path); - if (fileName !== METADATA_FILENAME) { - const templateVars = { - skillname: generateNameFromTitle(template.name), - generatedFiles, - template: template.template, - }; - let buf = await this.readFile(f.path); - /// Try not to template any non-text files. - if (isText(null, buf)) { - buf = Buffer.from(_.template(buf.toString(), { interpolate: /{{([\s\S]+?)}}/g })(templateVars)); - } - const relPath = f.path.slice(path.dirname(template.template.path).length); - const sourcePath = _.template(relPath, { interpolate: /__([\s\S]+?)__/g })(templateVars); - const targetPath = path.join(destinationPath, sourcePath); - await mkdir(path.dirname(targetPath), { recursive: true }); - await writeFile(targetPath, buf); - _.set(treeObj, sourcePath.split('/'), null); - } - } catch (err) { - printError(err.message, this.options); - } - })); + const treeObj = await this.generateTemplate(tree, template, destinationPath); + printToTerminal(''); if (!options || !boolean(options.notree)) { printSuccess('Generated the following files:'); - console.log(''); - console.log(destinationPath); - console.log(treeify.asTree(treeObj, true)); + printToTerminal(''); + printToTerminal(destinationPath); + printToTerminal(treeify.asTree(treeObj, true)); } - printSuccess('Workspace generation complete.', this.options); + printSuccess('Generation complete.', this.options); } } else { printError('Unable to retrieve templates', this.options); } } } + + +export class WorkspaceGenerateCommand extends TemplateGenerationCommand { + constructor(program) { + super(program, 'Skill', 'skills', 'skillname', 'templateConfig'); + } + + async configureSubcommand() { + await (new WorkspaceConfigureCommand(this.program)).execute({ refresh: true }); + } +} diff --git a/src/compatibility.js b/src/compatibility.js index cbd6a0a6..9fc55314 100644 --- a/src/compatibility.js +++ b/src/compatibility.js @@ -138,6 +138,7 @@ export function callCommand(fn) { } return fn(...args); } catch (err) { + debug('Error: %s\n\nStack: %s', err, err?.stack); handleError(err); } }; diff --git a/src/config.js b/src/config.js index 208d7133..7b0eb0e8 100644 --- a/src/config.js +++ b/src/config.js @@ -100,6 +100,7 @@ const ProfileSchemaV5 = Joi.object().keys({ currentRegistry: Joi.string().required(), templateConfig: Joi.any().required(), featureFlags: Joi.object().optional(), + pipelineTemplateConfig: Joi.any().optional(), }); /* eslint-disable no-param-reassign */ @@ -339,7 +340,7 @@ class ConfigV4 { } class ProfileV5 { constructor(name, { - url, username, issuer, audience, jwk, project, registries, currentRegistry, templateConfig, featureFlags, + url, username, issuer, audience, jwk, project, registries, currentRegistry, templateConfig, featureFlags, pipelineTemplateConfig, }, templateConfigV4) { this.name = name; this.url = url; @@ -360,6 +361,10 @@ class ProfileV5 { repo: 'CognitiveScale/cortex-code-templates', branch: 'main', }; + this.pipelineTemplateConfig = pipelineTemplateConfig || { + repo: 'CognitiveScale/cortex-code-templates', + branch: 'main', + }; this.featureFlags = featureFlags; } @@ -383,6 +388,7 @@ class ProfileV5 { registries: this.registries, currentRegistry: this.currentRegistry, templateConfig: this.templateConfig, + pipelineTemplateConfig: this.pipelineTemplateConfig, featureFlags: this.featureFlags, // TODO: is featureFlags needed as a field? }; }