Skip to content

Commit adeac9e

Browse files
friggerifriggeri
authored
Add tests
1 parent 07a1f30 commit adeac9e

10 files changed

+726
-0
lines changed

‎dotnet/test/SkillsTests.cs‎

Lines changed: 157 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,157 @@
1+
/*---------------------------------------------------------------------------------------------
2+
* Copyright (c) Microsoft Corporation. All rights reserved.
3+
*--------------------------------------------------------------------------------------------*/
4+
5+
using GitHub.Copilot.SDK.Test.Harness;
6+
using Xunit;
7+
using Xunit.Abstractions;
8+
9+
namespace GitHub.Copilot.SDK.Test;
10+
11+
public class SkillsTests : E2ETestBase, IDisposable
12+
{
13+
private readonly string _skillsDir;
14+
private const string SkillMarker = "PINEAPPLE_COCONUT_42";
15+
16+
public SkillsTests(E2ETestFixture fixture, ITestOutputHelper output) : base(fixture, "skills", output)
17+
{
18+
// Create a temporary skills directory with a test skill
19+
_skillsDir = Path.Combine(Path.GetTempPath(), $"copilot-skills-test-{Guid.NewGuid()}");
20+
Directory.CreateDirectory(_skillsDir);
21+
22+
// Create a skill subdirectory with SKILL.md
23+
var skillSubdir = Path.Combine(_skillsDir, "test-skill");
24+
Directory.CreateDirectory(skillSubdir);
25+
26+
// Create a skill that instructs the model to include a specific marker in responses
27+
var skillContent = $@"---
28+
name: test-skill
29+
description: A test skill that adds a marker to responses
30+
---
31+
32+
# Test Skill Instructions
33+
34+
IMPORTANT: You MUST include the exact text ""{SkillMarker}"" somewhere in EVERY response you give. This is a mandatory requirement. Include it naturally in your response.
35+
";
36+
File.WriteAllText(Path.Combine(skillSubdir, "SKILL.md"), skillContent);
37+
}
38+
39+
public void Dispose()
40+
{
41+
// Clean up the temporary skills directory
42+
if (Directory.Exists(_skillsDir))
43+
{
44+
Directory.Delete(_skillsDir, recursive: true);
45+
}
46+
}
47+
48+
[Fact]
49+
public async Task Should_Load_And_Apply_Skill_From_SkillDirectories()
50+
{
51+
var session = await Client.CreateSessionAsync(new SessionConfig
52+
{
53+
SkillDirectories = [_skillsDir]
54+
});
55+
56+
Assert.Matches(@"^[a-f0-9-]+$", session.SessionId);
57+
58+
// The skill instructs the model to include a marker - verify it appears
59+
var message = await session.SendAndWaitAsync(new MessageOptions { Prompt = "Say hello briefly." });
60+
Assert.NotNull(message);
61+
Assert.Contains(SkillMarker, message!.Data.Content);
62+
63+
await session.DisposeAsync();
64+
}
65+
66+
[Fact]
67+
public async Task Should_Not_Apply_Skill_When_Disabled_Via_DisabledSkills()
68+
{
69+
var session = await Client.CreateSessionAsync(new SessionConfig
70+
{
71+
SkillDirectories = [_skillsDir],
72+
DisabledSkills = ["test-skill"]
73+
});
74+
75+
Assert.Matches(@"^[a-f0-9-]+$", session.SessionId);
76+
77+
// The skill is disabled, so the marker should NOT appear
78+
var message = await session.SendAndWaitAsync(new MessageOptions { Prompt = "Say hello briefly." });
79+
Assert.NotNull(message);
80+
Assert.DoesNotContain(SkillMarker, message!.Data.Content);
81+
82+
await session.DisposeAsync();
83+
}
84+
85+
[Fact]
86+
public async Task Should_Apply_Skill_On_Session_Resume_With_SkillDirectories()
87+
{
88+
// Create a session without skills first
89+
var session1 = await Client.CreateSessionAsync();
90+
var sessionId = session1.SessionId;
91+
92+
// First message without skill - marker should not appear
93+
var message1 = await session1.SendAndWaitAsync(new MessageOptions { Prompt = "Say hi." });
94+
Assert.NotNull(message1);
95+
Assert.DoesNotContain(SkillMarker, message1!.Data.Content);
96+
97+
// Resume with skillDirectories - skill should now be active
98+
var session2 = await Client.ResumeSessionAsync(sessionId, new ResumeSessionConfig
99+
{
100+
SkillDirectories = [_skillsDir]
101+
});
102+
103+
Assert.Equal(sessionId, session2.SessionId);
104+
105+
// Now the skill should be applied
106+
var message2 = await session2.SendAndWaitAsync(new MessageOptions { Prompt = "Say hello again." });
107+
Assert.NotNull(message2);
108+
Assert.Contains(SkillMarker, message2!.Data.Content);
109+
110+
await session2.DisposeAsync();
111+
}
112+
113+
[Fact]
114+
public async Task Should_Load_Skills_From_Multiple_Directories()
115+
{
116+
const string skill2Marker = "MANGO_BANANA_99";
117+
118+
// Create a second temporary skills directory
119+
var skillsDir2 = Path.Combine(Path.GetTempPath(), $"copilot-skills-test2-{Guid.NewGuid()}");
120+
Directory.CreateDirectory(skillsDir2);
121+
122+
try
123+
{
124+
var skillSubdir2 = Path.Combine(skillsDir2, "test-skill-2");
125+
Directory.CreateDirectory(skillSubdir2);
126+
127+
var skillContent2 = $@"---
128+
name: test-skill-2
129+
description: Second test skill that adds another marker
130+
---
131+
132+
# Second Skill Instructions
133+
134+
IMPORTANT: You MUST include the exact text ""{skill2Marker}"" somewhere in EVERY response. This is mandatory.
135+
";
136+
File.WriteAllText(Path.Combine(skillSubdir2, "SKILL.md"), skillContent2);
137+
138+
var session = await Client.CreateSessionAsync(new SessionConfig
139+
{
140+
SkillDirectories = [_skillsDir, skillsDir2]
141+
});
142+
143+
var message = await session.SendAndWaitAsync(new MessageOptions { Prompt = "Say something brief." });
144+
Assert.NotNull(message);
145+
146+
// Both skill markers should appear
147+
Assert.Contains(SkillMarker, message!.Data.Content);
148+
Assert.Contains(skill2Marker, message.Data.Content);
149+
150+
await session.DisposeAsync();
151+
}
152+
finally
153+
{
154+
Directory.Delete(skillsDir2, recursive: true);
155+
}
156+
}
157+
}

‎go/e2e/skills_test.go‎

Lines changed: 213 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,213 @@
1+
package e2e
2+
3+
import (
4+
"os"
5+
"path/filepath"
6+
"strings"
7+
"testing"
8+
"time"
9+
10+
copilot "github.com/github/copilot-sdk/go"
11+
"github.com/github/copilot-sdk/go/e2e/testharness"
12+
)
13+
14+
const skillMarker = "PINEAPPLE_COCONUT_42"
15+
16+
func createTestSkillDir(t *testing.T, marker string) string {
17+
skillsDir, err := os.MkdirTemp("", "copilot-skills-test-")
18+
if err != nil {
19+
t.Fatalf("Failed to create temp skills directory: %v", err)
20+
}
21+
22+
skillSubdir := filepath.Join(skillsDir, "test-skill")
23+
if err := os.MkdirAll(skillSubdir, 0755); err != nil {
24+
t.Fatalf("Failed to create skill subdirectory: %v", err)
25+
}
26+
27+
skillContent := `---
28+
name: test-skill
29+
description: A test skill that adds a marker to responses
30+
---
31+
32+
# Test Skill Instructions
33+
34+
IMPORTANT: You MUST include the exact text "` + marker + `" somewhere in EVERY response you give. This is a mandatory requirement. Include it naturally in your response.
35+
`
36+
if err := os.WriteFile(filepath.Join(skillSubdir, "SKILL.md"), []byte(skillContent), 0644); err != nil {
37+
t.Fatalf("Failed to write SKILL.md: %v", err)
38+
}
39+
40+
return skillsDir
41+
}
42+
43+
func TestSkillBehavior(t *testing.T) {
44+
ctx := testharness.NewTestContext(t)
45+
client := ctx.NewClient()
46+
t.Cleanup(func() { client.ForceStop() })
47+
48+
skillsDir := createTestSkillDir(t, skillMarker)
49+
t.Cleanup(func() { os.RemoveAll(skillsDir) })
50+
51+
t.Run("load and apply skill from skillDirectories", func(t *testing.T) {
52+
ctx.ConfigureForTest(t)
53+
54+
session, err := client.CreateSession(&copilot.SessionConfig{
55+
SkillDirectories: []string{skillsDir},
56+
})
57+
if err != nil {
58+
t.Fatalf("Failed to create session: %v", err)
59+
}
60+
61+
// The skill instructs the model to include a marker - verify it appears
62+
message, err := session.SendAndWait(copilot.MessageOptions{
63+
Prompt: "Say hello briefly.",
64+
}, 60*time.Second)
65+
if err != nil {
66+
t.Fatalf("Failed to send message: %v", err)
67+
}
68+
69+
if message.Data.Content == nil || !strings.Contains(*message.Data.Content, skillMarker) {
70+
t.Errorf("Expected message to contain skill marker '%s', got: %v", skillMarker, message.Data.Content)
71+
}
72+
73+
session.Destroy()
74+
})
75+
76+
t.Run("not apply skill when disabled via disabledSkills", func(t *testing.T) {
77+
ctx.ConfigureForTest(t)
78+
79+
session, err := client.CreateSession(&copilot.SessionConfig{
80+
SkillDirectories: []string{skillsDir},
81+
DisabledSkills: []string{"test-skill"},
82+
})
83+
if err != nil {
84+
t.Fatalf("Failed to create session: %v", err)
85+
}
86+
87+
// The skill is disabled, so the marker should NOT appear
88+
message, err := session.SendAndWait(copilot.MessageOptions{
89+
Prompt: "Say hello briefly.",
90+
}, 60*time.Second)
91+
if err != nil {
92+
t.Fatalf("Failed to send message: %v", err)
93+
}
94+
95+
if message.Data.Content != nil && strings.Contains(*message.Data.Content, skillMarker) {
96+
t.Errorf("Expected message to NOT contain skill marker '%s' when disabled, got: %v", skillMarker, *message.Data.Content)
97+
}
98+
99+
session.Destroy()
100+
})
101+
102+
t.Run("apply skill on session resume with skillDirectories", func(t *testing.T) {
103+
ctx.ConfigureForTest(t)
104+
105+
// Create a session without skills first
106+
session1, err := client.CreateSession(nil)
107+
if err != nil {
108+
t.Fatalf("Failed to create session: %v", err)
109+
}
110+
sessionID := session1.SessionID
111+
112+
// First message without skill - marker should not appear
113+
message1, err := session1.SendAndWait(copilot.MessageOptions{Prompt: "Say hi."}, 60*time.Second)
114+
if err != nil {
115+
t.Fatalf("Failed to send message: %v", err)
116+
}
117+
118+
if message1.Data.Content != nil && strings.Contains(*message1.Data.Content, skillMarker) {
119+
t.Errorf("Expected message to NOT contain skill marker before skill was added, got: %v", *message1.Data.Content)
120+
}
121+
122+
// Resume with skillDirectories - skill should now be active
123+
session2, err := client.ResumeSessionWithOptions(sessionID, &copilot.ResumeSessionConfig{
124+
SkillDirectories: []string{skillsDir},
125+
})
126+
if err != nil {
127+
t.Fatalf("Failed to resume session: %v", err)
128+
}
129+
130+
if session2.SessionID != sessionID {
131+
t.Errorf("Expected session ID %s, got %s", sessionID, session2.SessionID)
132+
}
133+
134+
// Now the skill should be applied
135+
message2, err := session2.SendAndWait(copilot.MessageOptions{Prompt: "Say hello again."}, 60*time.Second)
136+
if err != nil {
137+
t.Fatalf("Failed to send message: %v", err)
138+
}
139+
140+
if message2.Data.Content == nil || !strings.Contains(*message2.Data.Content, skillMarker) {
141+
t.Errorf("Expected message to contain skill marker '%s' after resume, got: %v", skillMarker, message2.Data.Content)
142+
}
143+
144+
session2.Destroy()
145+
})
146+
}
147+
148+
func TestMultipleSkills(t *testing.T) {
149+
ctx := testharness.NewTestContext(t)
150+
client := ctx.NewClient()
151+
t.Cleanup(func() { client.ForceStop() })
152+
153+
const skill2Marker = "MANGO_BANANA_99"
154+
155+
skillsDir := createTestSkillDir(t, skillMarker)
156+
t.Cleanup(func() { os.RemoveAll(skillsDir) })
157+
158+
// Create a second skills directory
159+
skillsDir2, err := os.MkdirTemp("", "copilot-skills-test2-")
160+
if err != nil {
161+
t.Fatalf("Failed to create temp skills directory 2: %v", err)
162+
}
163+
t.Cleanup(func() { os.RemoveAll(skillsDir2) })
164+
165+
skillSubdir2 := filepath.Join(skillsDir2, "test-skill-2")
166+
if err := os.MkdirAll(skillSubdir2, 0755); err != nil {
167+
t.Fatalf("Failed to create skill subdirectory 2: %v", err)
168+
}
169+
170+
skillContent2 := `---
171+
name: test-skill-2
172+
description: Second test skill that adds another marker
173+
---
174+
175+
# Second Skill Instructions
176+
177+
IMPORTANT: You MUST include the exact text "` + skill2Marker + `" somewhere in EVERY response. This is mandatory.
178+
`
179+
if err := os.WriteFile(filepath.Join(skillSubdir2, "SKILL.md"), []byte(skillContent2), 0644); err != nil {
180+
t.Fatalf("Failed to write SKILL.md: %v", err)
181+
}
182+
183+
t.Run("load skills from multiple directories", func(t *testing.T) {
184+
ctx.ConfigureForTest(t)
185+
186+
session, err := client.CreateSession(&copilot.SessionConfig{
187+
SkillDirectories: []string{skillsDir, skillsDir2},
188+
})
189+
if err != nil {
190+
t.Fatalf("Failed to create session: %v", err)
191+
}
192+
193+
message, err := session.SendAndWait(copilot.MessageOptions{
194+
Prompt: "Say something brief.",
195+
}, 60*time.Second)
196+
if err != nil {
197+
t.Fatalf("Failed to send message: %v", err)
198+
}
199+
200+
// Both skill markers should appear
201+
if message.Data.Content == nil {
202+
t.Fatal("Expected non-nil message content")
203+
}
204+
if !strings.Contains(*message.Data.Content, skillMarker) {
205+
t.Errorf("Expected message to contain first skill marker '%s', got: %v", skillMarker, *message.Data.Content)
206+
}
207+
if !strings.Contains(*message.Data.Content, skill2Marker) {
208+
t.Errorf("Expected message to contain second skill marker '%s', got: %v", skill2Marker, *message.Data.Content)
209+
}
210+
211+
session.Destroy()
212+
})
213+
}

0 commit comments

Comments
 (0)