Skip to content

Commit 67e132e

Browse files
cbullingercbullinger
committed
Fix pagination and PR body template rendering in workflow processor
- Add pagination support to GetFilesChangedInPr() to handle PRs with >100 files - Update PullRequestQuery struct with PageInfo (EndCursor, HasNextPage) - Implement pagination loop to fetch all files across multiple pages - Update function signature to accept owner/repo parameters - Fix PR body template rendering in workflow processor - Add MessageTemplater to workflow processor dependencies - Render commit message, PR title, and PR body with MessageContext - Replace template variables with actual values (source_repo, pr_number, commit_sha) - Remove obsolete helper functions (getCommitMessage, getPRTitle, getPRBody) - Update webhook handler to pass MessageTemplater to workflow processor This fixes two critical issues: 1. Missing server files (js-express, python-fastapi) due to 100-file pagination limit 2. Blank PR descriptions with unrendered template variables
1 parent 83e8d8b commit 67e132e

File tree

10 files changed

+398
-41
lines changed

10 files changed

+398
-41
lines changed

‎examples-copier/code-copier‎

1.67 MB
Binary file not shown.

‎examples-copier/configs/env.yaml.example‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,10 @@ env_variables:
55

66
# GitHub App Configuration
77
GITHUB_APP_ID: "YOUR_GITHUB_APP_ID" # Your GitHub App ID (required)
8-
INSTALLATION_ID: "YOUR_INSTALLATION_ID" # GitHub App Installation ID (optional - can auto-discover)
8+
# INSTALLATION_ID is optional - used as fallback for legacy code
9+
# The app auto-discovers installation IDs per org using GetRestClientForOrg()
10+
# You can set this to any org's installation ID or leave it blank
11+
INSTALLATION_ID: "YOUR_INSTALLATION_ID" # GitHub App Installation ID (optional - auto-discovered per org)
912

1013
# Config Repository (where copier-config.yaml is stored)
1114
CONFIG_REPO_OWNER: "your-org" # Config repository owner (required)

‎examples-copier/configs/env.yaml.production‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,8 @@
22
# GitHub Configuration (Non-sensitive)
33
# =============================================================================
44
GITHUB_APP_ID: "1166559"
5+
# INSTALLATION_ID is optional - used as fallback for legacy code
6+
# The app auto-discovers installation IDs per org using GetRestClientForOrg()
57
INSTALLATION_ID: "62138132"
68

79
# Config Repository (where copier-config.yaml is stored)

‎examples-copier/copier-config-workflow.yaml‎

Lines changed: 224 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,224 @@
1+
# YAML configuration for the DevDocs GitHub Code Example Copier
2+
# See documentation at https://github.com/mongodb/code-example-tooling/tree/main/examples-copier/docs
3+
4+
# Global defaults (apply to all workflows unless overridden)
5+
defaults:
6+
commit_strategy:
7+
type: "pull_request"
8+
auto_merge: false
9+
deprecation_check:
10+
enabled: true
11+
file: "deprecated_examples.json"
12+
exclude:
13+
- "**/.env"
14+
- "**/.env.*"
15+
- "**/node_modules/**"
16+
- "**/__pycache__/**"
17+
18+
# ============================================================================
19+
# WORKFLOWS
20+
# ============================================================================
21+
# Each workflow defines:
22+
# - source: where files come from (repo + branch)
23+
# - destination: where files go to (repo + branch - automatically batched into PRs)
24+
# - transformations: how to copy/move files
25+
# ============================================================================
26+
27+
workflows:
28+
# ==========================================================================
29+
# MongoDB Org: docs-sample-apps → sample app repos
30+
# ==========================================================================
31+
32+
- name: "mflix-java"
33+
source:
34+
repo: "mongodb/docs-sample-apps"
35+
branch: "main"
36+
destination:
37+
repo: "mongodb/sample-app-java-mflix"
38+
branch: "main"
39+
transformations:
40+
- move: { from: "mflix/client", to: "client" }
41+
- move: { from: "mflix/server/java-spring", to: "server" }
42+
- copy: { from: "mflix/README-JAVA-SPRING.md", to: "README.md" }
43+
- copy: { from: "mflix/.gitignore-java", to: ".gitignore" }
44+
commit_strategy:
45+
type: "pull_request"
46+
pr_title: "Update MFlix client and Java server from docs-sample-apps"
47+
use_pr_template: true
48+
pr_body: |
49+
Automated update of MFlix Next.js client and Java Spring Boot server
50+
51+
Source: ${source_repo}
52+
PR: #${pr_number}
53+
Commit: ${commit_sha}
54+
auto_merge: false
55+
56+
- name: "mflix-nodejs"
57+
source:
58+
repo: "mongodb/docs-sample-apps"
59+
branch: "main"
60+
destination:
61+
repo: "mongodb/sample-app-nodejs-mflix"
62+
branch: "main"
63+
transformations:
64+
- move: { from: "mflix/client", to: "client" }
65+
- move: { from: "mflix/server/js-express", to: "server" }
66+
- copy: { from: "mflix/README-JAVASCRIPT-EXPRESS.md", to: "README.md" }
67+
- copy: { from: "mflix/.gitignore-js", to: ".gitignore" }
68+
commit_strategy:
69+
type: "pull_request"
70+
pr_title: "Update MFlix client and Express server from docs-sample-apps"
71+
use_pr_template: true
72+
pr_body: |
73+
Automated update of MFlix Next.js client and Express.js server
74+
75+
Source: ${source_repo}
76+
PR: #${pr_number}
77+
Commit: ${commit_sha}
78+
auto_merge: false
79+
exclude:
80+
- "\\.gitignore$"
81+
- "README.md$"
82+
83+
- name: "mflix-python"
84+
source:
85+
repo: "mongodb/docs-sample-apps"
86+
branch: "main"
87+
destination:
88+
repo: "mongodb/sample-app-python-mflix"
89+
branch: "main"
90+
transformations:
91+
- move: { from: "mflix/client", to: "client" }
92+
- move: { from: "mflix/server/python-fastapi", to: "server" }
93+
- copy: { from: "mflix/README-PYTHON-FASTAPI.md", to: "README.md" }
94+
- copy: { from: "mflix/.gitignore-python", to: ".gitignore" }
95+
commit_strategy:
96+
type: "pull_request"
97+
pr_title: "Update MFlix client and Python server from docs-sample-apps"
98+
use_pr_template: true
99+
pr_body: |
100+
Automated update of MFlix Next.js client and Python FastAPI server
101+
102+
Source: ${source_repo}
103+
PR: #${pr_number}
104+
Commit: ${commit_sha}
105+
auto_merge: false
106+
exclude:
107+
- "\\.gitignore$"
108+
- "README.md$"
109+
110+
# ==========================================================================
111+
# 10gen Org: Atlas SDK Go examples → Artifact repo
112+
# ==========================================================================
113+
114+
- name: "atlas-sdk-go-project-examples"
115+
source:
116+
repo: "10gen/docs-mongodb-internal"
117+
branch: "main"
118+
destination:
119+
repo: "mongodb/atlas-architecture-go-sdk"
120+
branch: "main"
121+
transformations:
122+
- glob:
123+
pattern: "content/code-examples/tested/go/atlas-sdk/project-copy/**/*"
124+
transform: "${relative_path}"
125+
commit_strategy:
126+
type: "pull_request"
127+
pr_title: "Update Atlas SDK Go examples from ${source_repo} PR ${pr_number}"
128+
pr_body: |
129+
Automated update of Atlas SDK Go project examples
130+
131+
**Source Details:**
132+
- Repository: ${source_repo}
133+
- Branch: ${source_branch}
134+
- Source PR: #${pr_number}
135+
- Commit: ${commit_sha}
136+
137+
**Changes:**
138+
- Files updated: ${file_count}
139+
auto_merge: false
140+
141+
# ==========================================================================
142+
# TEST: cbullinger Org: aggregation-tasks → sample app repos
143+
# ==========================================================================
144+
145+
- name: "test-mflix-java"
146+
source:
147+
repo: "cbullinger/aggregation-tasks"
148+
branch: "main"
149+
destination:
150+
repo: "cbullinger/sample-app-java-mflix"
151+
branch: "main"
152+
transformations:
153+
- move: { from: "mflix/client", to: "client" }
154+
- move: { from: "mflix/server/java-spring", to: "server" }
155+
- copy: { from: "mflix/README-JAVA-SPRING.md", to: "README.md" }
156+
- copy: { from: "mflix/.gitignore-java", to: ".gitignore" }
157+
commit_strategy:
158+
type: "pull_request"
159+
pr_title: "Update MFlix client and Java server from docs-sample-apps"
160+
use_pr_template: true
161+
pr_body: |
162+
Automated update of MFlix Next.js client and Java Spring Boot server
163+
164+
Source: ${source_repo}
165+
PR: #${pr_number}
166+
Commit: ${commit_sha}
167+
auto_merge: false
168+
exclude:
169+
- "\\.gitignore$"
170+
- "README.md$"
171+
172+
- name: "test-mflix-nodejs"
173+
source:
174+
repo: "cbullinger/aggregation-tasks"
175+
branch: "main"
176+
destination:
177+
repo: "cbullinger/sample-app-nodejs-mflix"
178+
branch: "main"
179+
transformations:
180+
- move: { from: "mflix/client", to: "client" }
181+
- move: { from: "mflix/server/js-express", to: "server" }
182+
- copy: { from: "mflix/README-JAVASCRIPT-EXPRESS.md", to: "README.md" }
183+
- copy: { from: "mflix/.gitignore-js", to: ".gitignore" }
184+
commit_strategy:
185+
type: "pull_request"
186+
pr_title: "Update MFlix client and Express server from docs-sample-apps"
187+
use_pr_template: true
188+
pr_body: |
189+
Automated update of MFlix Next.js client and Express.js server
190+
191+
Source: ${source_repo}
192+
PR: #${pr_number}
193+
Commit: ${commit_sha}
194+
auto_merge: false
195+
exclude:
196+
- "\\.gitignore$"
197+
- "README.md$"
198+
199+
- name: "test-mflix-python"
200+
source:
201+
repo: "cbullinger/aggregation-tasks"
202+
branch: "main"
203+
destination:
204+
repo: "cbullinger/sample-app-python-mflix"
205+
branch: "main"
206+
transformations:
207+
- move: { from: "mflix/client", to: "client" }
208+
- move: { from: "mflix/server/python-fastapi", to: "server" }
209+
- copy: { from: "mflix/README-PYTHON-FASTAPI.md", to: "README.md" }
210+
- copy: { from: "mflix/.gitignore-python", to: ".gitignore" }
211+
commit_strategy:
212+
type: "pull_request"
213+
pr_title: "Update MFlix client and Python server from docs-sample-apps"
214+
use_pr_template: true
215+
pr_body: |
216+
Automated update of MFlix Next.js client and Python FastAPI server
217+
218+
Source: ${source_repo}
219+
PR: #${pr_number}
220+
Commit: ${commit_sha}
221+
auto_merge: false
222+
exclude:
223+
- "\\.gitignore$"
224+
- "README.md$"

‎examples-copier/services/config_loader.go‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -75,8 +75,11 @@ func (cl *DefaultConfigLoader) LoadConfigFromContent(content string, filename st
7575

7676
// retrieveConfigFileContent fetches the config file content from the repository
7777
func retrieveConfigFileContent(ctx context.Context, filePath string, config *configs.Config) (string, error) {
78-
// Get GitHub client
79-
client := GetRestClient()
78+
// Get GitHub client for the config repo's org (auto-discovers installation ID)
79+
client, err := GetRestClientForOrg(config.ConfigRepoOwner)
80+
if err != nil {
81+
return "", fmt.Errorf("failed to get GitHub client for org %s: %w", config.ConfigRepoOwner, err)
82+
}
8083

8184
// Fetch file content
8285
fileContent, _, _, err := client.Repositories.GetContents(

‎examples-copier/services/github_read.go‎

Lines changed: 39 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -14,35 +14,54 @@ import (
1414

1515
// GetFilesChangedInPr retrieves the list of files changed in a specified pull request.
1616
// It returns a slice of ChangedFile structures containing details about each changed file.
17-
func GetFilesChangedInPr(pr_number int) ([]ChangedFile, error) {
17+
// Parameters:
18+
// - owner: The repository owner (e.g., "cbullinger")
19+
// - repo: The repository name (e.g., "aggregation-tasks")
20+
// - pr_number: The pull request number
21+
func GetFilesChangedInPr(owner string, repo string, pr_number int) ([]ChangedFile, error) {
1822
if InstallationAccessToken == "" {
1923
log.Println("No installation token provided")
2024
ConfigurePermissions()
2125
}
2226

23-
var prQuery PullRequestQuery
24-
variables := map[string]interface{}{
25-
"owner": githubv4.String(os.Getenv(configs.ConfigRepoOwner)),
26-
"name": githubv4.String(os.Getenv(configs.ConfigRepoName)),
27-
"number": githubv4.Int(pr_number),
28-
}
29-
3027
client := GetGraphQLClient()
3128
ctx := context.Background()
32-
err := client.Query(ctx, &prQuery, variables)
33-
if err != nil {
34-
LogCritical(fmt.Sprintf("Failed to execute query GetFilesChanged: %v", err))
35-
return nil, err
36-
}
3729

3830
var changedFiles []ChangedFile
39-
for _, edge := range prQuery.Repository.PullRequest.Files.Edges {
40-
changedFiles = append(changedFiles, ChangedFile{
41-
Path: string(edge.Node.Path),
42-
Additions: int(edge.Node.Additions),
43-
Deletions: int(edge.Node.Deletions),
44-
Status: string(edge.Node.ChangeType),
45-
})
31+
var cursor *githubv4.String = nil
32+
hasNextPage := true
33+
34+
// Paginate through all files
35+
for hasNextPage {
36+
var prQuery PullRequestQuery
37+
variables := map[string]interface{}{
38+
"owner": githubv4.String(owner),
39+
"name": githubv4.String(repo),
40+
"number": githubv4.Int(pr_number),
41+
"cursor": cursor,
42+
}
43+
44+
err := client.Query(ctx, &prQuery, variables)
45+
if err != nil {
46+
LogCritical(fmt.Sprintf("Failed to execute query GetFilesChanged: %v", err))
47+
return nil, err
48+
}
49+
50+
// Append files from this page
51+
for _, edge := range prQuery.Repository.PullRequest.Files.Edges {
52+
changedFiles = append(changedFiles, ChangedFile{
53+
Path: string(edge.Node.Path),
54+
Additions: int(edge.Node.Additions),
55+
Deletions: int(edge.Node.Deletions),
56+
Status: string(edge.Node.ChangeType),
57+
})
58+
}
59+
60+
// Check if there are more pages
61+
hasNextPage = prQuery.Repository.PullRequest.Files.PageInfo.HasNextPage
62+
if hasNextPage {
63+
cursor = &prQuery.Repository.PullRequest.Files.PageInfo.EndCursor
64+
}
4665
}
4766

4867
LogInfo(fmt.Sprintf("PR has %d changed files.", len(changedFiles)))

‎examples-copier/services/webhook_handler_new.go‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -277,8 +277,8 @@ func handleMergedPRWithContainer(ctx context.Context, prNumber int, sourceCommit
277277
return
278278
}
279279

280-
// Get changed files from PR
281-
changedFiles, err := GetFilesChangedInPr(prNumber)
280+
// Get changed files from PR (from the source repository that triggered the webhook)
281+
changedFiles, err := GetFilesChangedInPr(repoOwner, repoName, prNumber)
282282
if err != nil {
283283
LogAndReturnError(ctx, "get_files", "failed to get changed files", err)
284284
container.MetricsCollector.RecordWebhookFailed()
@@ -460,6 +460,7 @@ func processFilesWithWorkflows(ctx context.Context, prNumber int, sourceCommitSH
460460
container.PathTransformer,
461461
container.FileStateService,
462462
container.MetricsCollector,
463+
container.MessageTemplater,
463464
)
464465

465466
// Process each workflow

0 commit comments

Comments
 (0)