Fab 6046: Add Pipeline generation and repository configuration based on workspaces

README.md

@@ -134,7 +134,7 @@ Executables are created for the following platforms:
 
 ## Subcommands & Compatibility Checks
 
-The following sequene diagram shows an example running a command with  Cortex CLI - namely which HTTP requests. ([This diagram can be viewered here](https://mermaid.live/view#pako:eNp9VE1z2jAQ_Ss7nh6aAeJLT54OHcZAQ4c4TEx64iLLa6IiS6ok0zCZ_PdKlg0k0Ookad--_XgrvUZUlhglkcHfDQqKU0a2mtQbAW6RxkrR1AXqcJ5QKzVMcR-OhXyBdLmAVNY1EaUJt34poi2jTBFhIQVigEpt8eU6IG-KEwQUU8iZwI4NRRk25x5zxtEcjMX60pZ7rhyFITBZLS7tmbdnq_tgcbWMxuNBmlyEB86M3YgAo5o5AsJhKUkJKy0rl0Ewpd7_lFACj7PJFD7dPdzP4ttAGlMpKrYN-BO0C5zyxp00PD0uh576F1I7BLT0to_ehsgTuFuvV_B9toa4IoVmNN5_iZmoZEDlHd-P_CGDzznqvSNdsxoht6RWQ5gjsY1GmHOyNTc99ah1WrKatUrQTksoiMESpICqc6u825ko3YhwCwtz5um2Svmyy28nuX0FDpJAiRy3xCJYCebo07Fyg-9dBk6eBGZaS53Ak9gJ-Uf003aRSSYdrfRFu0jDzAnRtGBFLCsYZ_YA6TPS3YWmVzFtR5vC53298fTcKyZKcUdmmRQmDqqPKGdnyrTlB20e3Vtj2nX3J2rjPG76lELAQXYWsWti9oFisieMk4Jj-wA7nl7UfzYlD01xz9ihL2fbDTys3j-__7ZAhWE18dduN4775zPSqKRh7rtgR6oPJRwjnbKOhlGNuiasdD_Sq7_eRPYZa9xEiduWRO820Ua8OZz_mvKDoFFidYPDqFGlG6vu9wqXb38B-i2SbA)).
+The following sequence diagram shows an example running a command with  Cortex CLI - namely which HTTP requests. ([This diagram can be viewered here](https://mermaid.live/view#pako:eNp9VE1z2jAQ_Ss7nh6aAeJLT54OHcZAQ4c4TEx64iLLa6IiS6ok0zCZ_PdKlg0k0Ookad--_XgrvUZUlhglkcHfDQqKU0a2mtQbAW6RxkrR1AXqcJ5QKzVMcR-OhXyBdLmAVNY1EaUJt34poi2jTBFhIQVigEpt8eU6IG-KEwQUU8iZwI4NRRk25x5zxtEcjMX60pZ7rhyFITBZLS7tmbdnq_tgcbWMxuNBmlyEB86M3YgAo5o5AsJhKUkJKy0rl0Ewpd7_lFACj7PJFD7dPdzP4ttAGlMpKrYN-BO0C5zyxp00PD0uh576F1I7BLT0to_ehsgTuFuvV_B9toa4IoVmNN5_iZmoZEDlHd-P_CGDzznqvSNdsxoht6RWQ5gjsY1GmHOyNTc99ah1WrKatUrQTksoiMESpICqc6u825ko3YhwCwtz5um2Svmyy28nuX0FDpJAiRy3xCJYCebo07Fyg-9dBk6eBGZaS53Ak9gJ-Uf003aRSSYdrfRFu0jDzAnRtGBFLCsYZ_YA6TPS3YWmVzFtR5vC53298fTcKyZKcUdmmRQmDqqPKGdnyrTlB20e3Vtj2nX3J2rjPG76lELAQXYWsWti9oFisieMk4Jj-wA7nl7UfzYlD01xz9ihL2fbDTys3j-__7ZAhWE18dduN4775zPSqKRh7rtgR6oPJRwjnbKOhlGNuiasdD_Sq7_eRPYZa9xEiduWRO820Ua8OZz_mvKDoFFidYPDqFGlG6vu9wqXb38B-i2SbA)).
 
 The Cortex CLI will limit the set of available subcommands, to those which are supported by the Cortex cluster that it is
 communicating with.

bin/cortex-pipelines.js

@@ -7,6 +7,8 @@ import {
   RunPipelineCommand,
   DescribePipelineRunCommand,
   ListPipelineRunsCommand,
+  PipelineTemplateConfigureCommand,
+  PipelineGenerateCommand,
 } from '../src/commands/pipelines.js';
 import { callCommand } from '../src/compatibility.js';
 import {
@@ -110,6 +112,28 @@ export function create() {
       return new ListPipelineRunsCommand(pipelines).execute(pipelineName, gitRepoName, options);
   }));
 
+
+  // Configure Pipeline Template Github Repository for the current CLI Profile (does not support --profile)
+  pipelines
+    .command('configure')
+    .option('--color [boolean]', 'Turn on/off colors', 'true')
+    .option('--refresh', 'Refresh the Github access token')
+    .description('Configure the Cortex Template system for generating Pipeline templates')
+    .action(callCommand((options) => new PipelineTemplateConfigureCommand(pipelines).execute(options)));
+
+  // Generate a Pipeline template
+  pipelines.command('generate [pipelineName] [destination]')
+    .option('--profile <profile>', 'The profile to use')
+    .option('--color [boolean]', 'Turn on/off colors', 'true')
+    .option('--notree [boolean]', 'Do not display generated file tree', 'false')
+    .option('--template <templateName>', 'Name of the template to use')
+    .description('Generates a folder based on a Pipeline template from the template repository')
+    .action(callCommand((pipelineName, destination, options) => {
+      checkForEmptyArgs({ pipelineName, destination });
+      PipelineGenerateCommand.validatePipelineName(pipelineName, options);
+      return new PipelineGenerateCommand(pipelines).execute(pipelineName, destination, options);
+    }));
+
   return pipelines;
 }
 

bin/cortex-workspaces.js

@@ -1,8 +1,8 @@
 import { Command } from 'commander';
 import process from 'node:process';
 import esMain from 'es-main';
-import WorkspaceConfigureCommand from '../src/commands/workspaces/configure.js';
-import WorkspaceGenerateCommand from '../src/commands/workspaces/generate.js';
+import { WorkspaceConfigureCommand } from '../src/commands/workspaces/configure.js';
+import { WorkspaceGenerateCommand } from '../src/commands/workspaces/generate.js';
 import WorkspaceBuildCommand from '../src/commands/workspaces/build.js';
 import WorkspacePublishCommand from '../src/commands/workspaces/publish.js';
 import {
@@ -11,9 +11,10 @@ import {
 
 export function create() {
     const program = new Command();
-
     program.name('cortex workspaces');
     program.description('Scaffolding Cortex Components');
+
+    // Configure Template Github Repository for the current CLI Profile (does not support --profile)
     program
         .command('configure')
         .option('--refresh', 'Refresh the Github access token')

docs/workspaces.md

@@ -0,0 +1,237 @@
+# Workspaces
+
+## Overview
+
+Note that `workspaces` CLI commands are narrowly focused on generating Skill templates. Ideally `workspaces` would encompass Pipeline as well, but the initial implementation for Pipelines uses its CLI commands (similar to `workspaces`).
+
+Both commands only support Git Repositories hosted on GitHub!
+
+### Templates
+
+The primary repository with templates can be found at: <https://github.com/CognitiveScale/cortex-code-templates>
+
+* A template is any folder within the git Repository that contains a `metadata.json` file.
+
+    Expected structure for `metadata.json`:
+
+    ```json
+    {
+        "name": "<template-name>",
+        "title": "<template-title>",
+        "description": "<template-description>",
+        "tags": ["list", "of", "tags", "applied", "to", "template"],
+        "enabled": true,
+        "resourceType": "Skill" // e.g. "Pipeline"
+    }
+    ```
+
+* Templates can be nested in any subfolder in the repository
+* The templating process works via the [loadsh template](https://docs-lodash.com/v4/template/) util. Refer to that documentation for how variables are templated.
+
+### Additional Reference
+
+* [Example Cortex Development Workflow](https://drive.google.com/file/d/1tPyuqtNFz9JFtJuQE6HRxou_SLKIK8fO/view?usp=drive_link)
+* [Relation to Skill Building](https://docs.google.com/presentation/d/1k4vJ7d5oGbvFaUezl5dBXHtKc-CZK8IAGlvIq-RpJrk/edit#slide=id.g12c69d77b32_0_42) (i.e. migration from traditional Skill Building)
+
+## Workspaces Configure
+
+The configuration process implements the [Github OAuth Device Flow](https://docs.github.com/en/apps/oauth-apps/building-oauth-apps/authorizing-oauth-apps#device-flow). The reason for implementing this flow is primarily to avoid rate limiting issues from the GitHub rest API, see:
+
+* [Unauthenticated User Rate Limiting](https://docs.github.com/en/rest/using-the-rest-api/rate-limits-for-the-rest-api?apiVersion=2022-11-28#primary-rate-limit-for-unauthenticated-users) (~60 requests per hour)
+* [Authenticated User Rate Limiting](https://docs.github.com/en/rest/using-the-rest-api/rate-limits-for-the-rest-api?apiVersion=2022-11-28#primary-rate-limit-for-authenticated-users) (~5000 requests per hour)
+
+The GitHub authentication does NOT allow users to access private repositories. Using a private repository would require [registering the Cortex CLI App in your developer settings](https://github.com/orgs/community/discussions/48102).
+
+```mermaid
+%% Title: Cortex Workspaces configure CLI command
+sequenceDiagram
+    autonumber
+    title Workspaces Configure CLI subcommand 
+    Actor D as Dev
+    participant C as Cortex CLI
+    participant F as Filesystem
+
+    %% Mac: KeyChain
+    %% Linux: Secret Service API/libsecret
+    %% Windows: Credential Vault
+    participant K as System Keychain<br>(Optional)
+
+    participant B as Browser
+    participant G as GitHub
+
+    D->>+C: Configure CLI
+    Note over D,C: cortex workspaces configure --refresh --no-timeout
+
+    C->>+F: Read Config File<br>fetch existing config
+    C->>+D: Prompt user for Repository/Branch
+    C->>+D: User responds with answers
+
+    %% TODO: Check if token is valid
+    %% If it's invalid and not being refreshed - exit with error
+
+    C->>+B: Browser Opens at GitHub
+    D->>+B: Enter code & authorize Cortex CLI
+    loop Fixed Interval (e.g. 5 seconds)
+        C->>+G: Fetch GitHub Device Code
+        G->>+C: JSON (Potential Device Code)
+        Note over C,G: HHTP POST https://github.com/login/device/code
+
+        critical Check Access Token
+        option Access Token returned
+            alt Keychain service available
+                C->>+K: Store token in Keychain
+            else Local File fallback
+                C->>+F: Store token as file in Config folder
+            end
+            C->>+D: Configuration Successful
+        option Authorization Pending
+            C->>+C: Update Expiration Time
+        option Slow Down
+            C->>+C: Increment Poll Interval
+        option Expired Token
+            C->>+D: Exit with error - re-configure and try again
+        option Incorrect Device Code
+            C->>+D: Exit with error - incorrect device code entered
+        option Access Denied
+            C->>+D: Exit with error - Access denied by user
+        option Unexpected Error
+            C->>+D: Exit with error
+        end
+    end
+```
+
+<!--
+TODO's:
+- Work on pipeline template
+- Export data for Pipeline template to local CSV
+- Add example README
+- Test that workspaces generate command still works
+-->
+
+## Workspaces Generate
+
+The following diagram shows the sequence of steps taken when generating a template.
+
+```mermaid
+sequenceDiagram
+    autonumber
+    title Workspaces Generate CLI subcommand 
+    Actor D as Dev
+    participant C as Cortex CLI
+    participant F as Filesystem
+
+    %% Mac: KeyChain
+    %% Linux: Secret Service API/libsecret
+    %% Windows: Credential Vault
+    participant K as System Keychain<br>(Optional)
+    participant G as GitHub
+
+    D->>+C: Generate Pipeline
+    Note over D,C: cortex workspaces generate <name> <destination>
+
+    critical Github Token Validation
+        C->>+F: Read the user config File
+        alt Keychain service available
+            C->>+K: Load token from Keychain
+        else Local File fallback
+            C->>+F: Load token from cached file in Config folder
+        end
+        %% Valdiate the Token 
+        C->>+G: Validate Token with Github API
+        Note over C,G: HTTP GET https://api.github.com/user (with Token)
+
+        option Token is Invalid
+            C->>+C: Force user to configure template repository (recreate Token)
+        option Token is Valid
+            C->>+C: Continue (No-Op)
+    end
+
+    opt Fail early if destination already exists
+        C->>+F: Check if the destination exists
+        C->>+D: Report Error to user
+    end
+
+    critical Fetch Git Tree for the Repository/Branch
+        C->>+G: Fetch the HEAD of the configured repository/branch
+        G->>+C: git SHA (JSON)
+        Note over C,G: HTTP GET https://api.github.com/repos/<repo>/branches/<branch>
+        alt Repository & Branch exist
+            C->>+G: Fetch Git Tree(s) from the Repo
+            G->>+C: Git Tree (JSON)/fetch
+            Note over C,G: HTTP GET https://api.github.com/repos/<repo>/git/trees/<sha>?recursive=true
+        end
+    end
+
+    critical Select Template
+        Note right of C: Templates are identifed based<br>on presence of `metadata.json`
+        C->>+C: Use glob to find potential templates in Git Tree
+        loop For each potential template
+            C->>+G: Read the metadata.json file for the Template
+            G->>+C: JSON
+            Note over C,G: HTTP GET https://api.github.com/repos/<repo>/contents/<filePath>?ref=<branch>
+        end
+
+        C->>+D: Prompt User for template selection
+        D->>+C: Answers (Template Choice, Template Name, etc.)
+        alt No Templates found
+            C->>+D: Display Error
+        end
+        alt Selected Template does not exist
+            C->>+D: Display Error
+        end
+    end
+
+    critical Generate Template
+        C->>+C: Use glob to identify files corresponding to template in Git Tree
+
+        opt Fail early if destination already exists
+            C->>+F: Check if the destination exists
+            C->>+D: Report Error to user
+        end
+
+        C->>+C: Compute generated files<br>with templated values
+        loop For each templated file
+            C->>+G: Fetch file contents
+            G->>+C: File Contents (stream)
+            Note over C,G: HTTP GET https://api.github.com/repos/<repo>/contents/<filePath>?ref=<branch>
+            C->>+C: Apply templating
+            C->>+F: Write templated file to Filesystem
+        end
+    end
+
+    opt Display generated file tree to user
+        C->>+D: Print the generated file tree 
+    end
+```
+
+## Relation to Pipelines
+
+The Cortex CLI offers similar capabilities for generating Pipelines template, similar to Workspaces, however, the both are distinct features (subcommands) in the CLI.
+
+### Pipelines Configure
+
+The process is simlilar to that of [workspaces configure](#workspaces-configure), but:
+
+* A separate section of the Config file is used to store the Repository configuration - i.e. The Pipeline Template repository can be different
+
+```bash
+$ cortex pipelines configure --refresh --no-timeout
+Configuring workspaces for profile qa-aks
+? Template Repository URL:    CognitiveScale/cortex-code-templates
+? Template Repository Branch: FAB-6046-pipeline-generate
+Opening browser at https://github.com/login/device
+Please enter the following code to authorize the Cortex CLI: FB05-0378   ( Expires in 14 minutes and 58 seconds ) - CTRL-C to abort
+Github token configuration successful.
+```
+
+### Pipelines Generate
+
+The process is simlilar to that of [workspaces generate](#workspaces-generate), but only Pipeline templates are available.
+
+```bash
+$ cortex pipelines generate
+```
+
+## Workspaces vs Pipelines
+
+The Cortex CLI offers similar capabilities for generating Pipelines template, similar to Workspaces, however, the both are distinct features (subcommands) in the CLI.

src/commands/pipelines.js

@@ -1,6 +1,7 @@
 import debugSetup from 'debug';
 import fs from 'fs';
 import dayjs from 'dayjs';
+import path from 'path';
 import _ from 'lodash';
 import relativeTime from 'dayjs/plugin/relativeTime.js';
 import { loadProfile } from '../config.js';
@@ -17,12 +18,14 @@ import {
   printTable,
   printWarning, handleError,
 } from './utils.js';
+import { TemplateConfigureCommand } from './workspaces/configure.js';
+import { TemplateGenerationCommand } from './workspaces/generate.js';
+
 
 const debug = debugSetup('cortex:cli');
 dayjs.extend(relativeTime);
 
 
-
 export const ListPipelineCommand = class {
   constructor(program) {
     this.program = program;
@@ -241,3 +244,49 @@ export const ListPipelineRunsCommand = class {
     }
   }
 };
+
+// NOTE: Easiest way to piggy-back of the existing functionality from Workspaces is to
+// directly use the same logic via inheritance.
+//
+// The constructor assigns a different configKey to avoid collision from sharing the
+// same property in the config file.
+export const PipelineTemplateConfigureCommand = class extends TemplateConfigureCommand {
+  constructor(program) {
+    super(program, 'pipelineTemplateConfig', 'Pipelines');
+  }
+};
+
+export const PipelineGenerateCommand = class extends TemplateGenerationCommand {
+  constructor(program) {
+    super(program, 'Pipeline', 'pipelines', 'pipelinename', 'pipelineTemplateConfig');
+  }
+
+  shouldTemplateFile(filepath) {
+    // NOTE: It is common for dbt's templating (jinja2?) to collide with Lodash's templating syntax.
+    // Current workaround is to exclude specific DBT & SQL files from templating
+    const fileName = path.posix.basename(filepath);
+    if (fileName === 'dbt_project.yml' || fileName === 'dbt_project.yaml') {
+      // Allow templating in the main dbt project file
+      return true;
+    }
+    if (path.extname(fileName) === '.sql' || filepath.includes('/dbt_packages/') || filepath.includes('/dbt')) {
+      return false;
+    }
+    return true;
+  }
+
+  async configureSubcommand() {
+    await (new PipelineTemplateConfigureCommand(this.program)).execute({ refresh: true });
+  }
+
+  static validatePipelineName(pipelineName, options) {
+    // Pipelines must have a name that is only lowercase characters, numbers, and underscore.
+    // Need to validate this before hand, otherwise the generated template will be unusable.
+    const nameRegex = /^[a-z](_?[a-z0-9]+)*$/g; // Taken from 'sensa-data-pipelines' package
+    if (!pipelineName || !nameRegex.test(pipelineName)) {
+      // Print error & exit
+      printError(`Cannot generate Pipeline with name "${pipelineName}". Pipeline names must conform to the regex: `
+        + `${nameRegex} (all lowercase characters, with numbers or underscores)`, options);
+    }
+  }
+};

src/commands/utils.js

@@ -529,9 +529,9 @@ export function printErrorDetails(response, options, exit = true) {
 }
 
 function checkForEmptyString(key, value) {
-    if (!value.trim()) {
-        printError(`error: <${key}> cannot be empty.`);
-        process.exit(1); // Exit with an error code
+    if (!(value?.trim())) {
+      printError(`error: <${key}> cannot be empty.`);
+      process.exit(1); // Exit with an error code
     }
 }