Skip to content

Commit

Permalink
AAP-32136: VS Code Extension: Playbook custom_prompt titivation (#1554)
Browse files Browse the repository at this point in the history
  • Loading branch information
@manstis
manstis authored Sep 27, 2024
1 parent 2bceee6 commit 225e0d6
Show file tree
Hide file tree
Showing 2 changed files with 9 additions and 129 deletions.
8 changes: 6 additions & 2 deletions package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -648,13 +648,17 @@
"ansible.lightspeed.playbookGenerationCustomPrompt": {
"scope": "resource",
"type": "string",
"markdownDescription": "Custom Prompt for Playbook generation. This setting is only applicable to commercial users with an Ansible Lightspeed seat assignment.",
"markdownDescription": "Custom Prompt for Playbook generation.",
"pattern": "^$|^.*{goal}.*{outline}.*|.*{outline}.*{goal}.*$",
"patternErrorMessage": "The prompt must contain placeholders for both '{goal}' and '{outline}'.",
"order": 5
},
"ansible.lightspeed.playbookExplanationCustomPrompt": {
"scope": "resource",
"type": "string",
"markdownDescription": "Custom Prompt for Playbook explanation. This setting is only applicable to commercial users with an Ansible Lightspeed seat assignment.",
"markdownDescription": "Custom Prompt for Playbook explanation.",
"pattern": "^$|^.*{playbook}.*$",
"patternErrorMessage": "The prompt must contain a placeholder for '{playbook}'.",
"order": 6
}
}
Expand Down
130 changes: 3 additions & 127 deletions test/ui-test/lightspeedUiTest.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -638,7 +638,7 @@ export function lightspeedUIAssetsTest(): void {
await updateSettings(
settingsEditor,
"ansible.lightspeed.playbookGenerationCustomPrompt",
"custom prompt",
"custom prompt {goal}, {outline}",
);
await workbench.executeCommand("View: Close All Editor Groups");

Expand Down Expand Up @@ -694,68 +694,6 @@ export function lightspeedUIAssetsTest(): void {
}
});

it("Playbook generation webview works as expected (fast path, custom prompt, broken)", async function () {
// Execute only when TEST_LIGHTSPEED_URL environment variable is defined.
if (process.env.TEST_LIGHTSPEED_URL) {
// Set Playbook generation custom prompt
settingsEditor = await workbench.openSettings();
await updateSettings(
settingsEditor,
"ansible.lightspeed.playbookGenerationCustomPrompt",
"custom prompt broken",
);
await workbench.executeCommand("View: Close All Editor Groups");

// Open playbook generation webview.
await workbench.executeCommand(
"Ansible Lightspeed: Playbook generation",
);
await sleep(2000);
const webView = await new WebView();
expect(webView, "webView should not be undefined").not.to.be.undefined;
await webView.switchToFrame(5000);
expect(
webView,
"webView should not be undefined after switching to its frame",
).not.to.be.undefined;

// Set input text and invoke summaries API
const textArea = await webView.findWebElement(
By.xpath("//vscode-text-area"),
);
expect(textArea, "textArea should not be undefined").not.to.be
.undefined;
const submitButton = await webView.findWebElement(
By.xpath("//vscode-button[@id='submit-button']"),
);
expect(submitButton, "submitButton should not be undefined").not.to.be
.undefined;
//
// Note: Following line should succeed, but fails for some unknown reasons.
//
// expect((await submitButton.isEnabled()), "submit button should be disabled by default").is.false;
await textArea.sendKeys("Create an azure network.");
expect(
await submitButton.isEnabled(),
"submit button should be enabled now",
).to.be.true;
await submitButton.click();
await sleep(2000);

await webView.switchBack();

const notifications = await workbench.getNotifications();
const notification = notifications[0];
expect(await notification.getMessage()).equals(
"Bad Request response. Please try again. customPrompt: custom prompt is invalid",
);

await workbench.executeCommand("View: Close All Editor Groups");
} else {
this.skip();
}
});

it("Playbook explanation webview works as expected", async function () {
if (process.env.TEST_LIGHTSPEED_URL) {
const folder = "lightspeed";
Expand Down Expand Up @@ -927,7 +865,7 @@ export function lightspeedUIAssetsTest(): void {
await updateSettings(
settingsEditor,
"ansible.lightspeed.playbookExplanationCustomPrompt",
"custom prompt",
"custom prompt {playbook}",
);
await workbench.executeCommand("View: Close All Editor Groups");

Expand Down Expand Up @@ -981,7 +919,7 @@ export function lightspeedUIAssetsTest(): void {
await updateSettings(
settingsEditor,
"ansible.lightspeed.playbookExplanationCustomPrompt",
"custom prompt",
"custom prompt {playbook}",
);
await workbench.executeCommand("View: Close All Editor Groups");

Expand Down Expand Up @@ -1022,68 +960,6 @@ export function lightspeedUIAssetsTest(): void {
}
});

it("Playbook explanation webview works as expected, custom prompt, broken", async function () {
if (process.env.TEST_LIGHTSPEED_URL) {
const folder = "lightspeed";
const file = "playbook_4.yml";
const filePath = getFixturePath(folder, file);

// Set Playbook generation custom prompt
settingsEditor = await workbench.openSettings();
await updateSettings(
settingsEditor,
"ansible.lightspeed.playbookExplanationCustomPrompt",
"custom prompt broken",
);
await workbench.executeCommand("View: Close All Editor Groups");

// Open file in the editor
await VSBrowser.instance.openResources(filePath);

// Open playbook explanation webview.
await workbench.executeCommand(
"Explain the playbook with Ansible Lightspeed",
);
await sleep(2000);

// Locate the playbook explanation webview
const webView = (await new EditorView().openEditor(
"Explanation",
1,
)) as WebView;
expect(webView, "webView should not be undefined").not.to.be.undefined;
await webView.switchToFrame(5000);
expect(
webView,
"webView should not be undefined after switching to its frame",
).not.to.be.undefined;

// Find the main div element of the webview and verify the expected text is found.
const mainDiv = await webView.findWebElement(
By.xpath("//div[contains(@class, 'playbookGeneration') ]"),
);
expect(mainDiv, "mainDiv should not be undefined").not.to.be.undefined;
const text = await mainDiv.getText();
expect(
text.includes(
"Bad Request response. Please try again. customPrompt: custom prompt is invalid",
),
).to.be.true;

await webView.switchBack();

const notifications = await workbench.getNotifications();
const notification = notifications[0];
expect(await notification.getMessage()).equals(
"Bad Request response. Please try again. customPrompt: custom prompt is invalid",
);

await workbench.executeCommand("View: Close All Editor Groups");
} else {
this.skip();
}
});

after(async function () {
if (process.env.TEST_LIGHTSPEED_URL) {
settingsEditor = await workbench.openSettings();
Expand Down

0 comments on commit 225e0d6

Please sign in to comment.