Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Add exception handling in GitHub Project fetch job #228

Merged
merged 2 commits into from
Oct 13, 2024
Merged

Add exception handling in GitHub Project fetch job #228

Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
572234a
Add try-catch block for GitHub Project fetch job
dexter-sim Oct 13, 2024
4ddea6c
Merge branch 'staging' into bugfixes-2
dexter-sim Oct 13, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
340 changes: 174 additions & 166 deletions backend/jobs/githubJob.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -269,14 +269,15 @@ const fetchGitHubProjectData = async (
let projectEndCursor = '';
let gitHubProjectCount = 0;

while (hasNextProject && gitHubProjectCount < 25) {
let hasNextIssue = true;
let issueEndCursor = '';
gitHubProjectCount++;

while (hasNextIssue) {
const data: any = await octokit.graphql(
`query {
try {
while (hasNextProject && gitHubProjectCount < 25) {
let hasNextIssue = true;
let issueEndCursor = '';
gitHubProjectCount++;

while (hasNextIssue) {
const data: any = await octokit.graphql(
`query {
organization(login: "${gitHubOrgName}") {
projectsV2(first: 1, orderBy: {field: UPDATED_AT, direction: DESC}, after: "${projectEndCursor}") {
pageInfo {
Expand Down Expand Up @@ -357,198 +358,205 @@ const fetchGitHubProjectData = async (
}
}
}`
);
);

projectEndCursor = data.organization.projectsV2.pageInfo.endCursor;
hasNextProject = data.organization.projectsV2.pageInfo.hasNextPage;
issueEndCursor =
data.organization.projectsV2.nodes[0].items.pageInfo.endCursor;
hasNextIssue =
data.organization.projectsV2.nodes[0].items.pageInfo.hasNextIssue;

// Transform the fetched data into the required schema format
const transformedProjects = data.organization.projectsV2.nodes.map(
(project: any) => ({
id: project.id,
title: project.title,
gitHubOrgName: gitHubOrgName,
items: project.items.nodes.map((item: any) => ({
type: item.type,
content: {
id: item.content.id,
title: item.content.title,
url: item.content.url,
closed: item.content.closed,
labels: item.content.labels?.nodes.map((label: any) => ({
name: label.name,
})),
milestone: item.content.milestone
? {
id: item.content.milestone.id,
title: item.content.milestone.title,
description: item.content.milestone.description,
number: item.content.milestone.number,
createdAt: item.content.milestone.createdAt,
dueOn: item.content.milestone.dueOn,
state: item.content.milestone.state,
}
: null,
assignees: item.content.assignees
? item.content.assignees.nodes.map((assignee: any) => ({
id: assignee.id,
login: assignee.login,
name: assignee.name,
}))
: [],
__typename: item.content.__typename,
},
fieldValues: item.fieldValues.nodes
.filter((field: any) => field.field) // Filter out empty objects
projectEndCursor = data.organization.projectsV2.pageInfo.endCursor;
hasNextProject = data.organization.projectsV2.pageInfo.hasNextPage;
issueEndCursor =
data.organization.projectsV2.nodes[0].items.pageInfo.endCursor;
hasNextIssue =
data.organization.projectsV2.nodes[0].items.pageInfo.hasNextIssue;

// Transform the fetched data into the required schema format
const transformedProjects = data.organization.projectsV2.nodes.map(
(project: any) => ({
id: project.id,
title: project.title,
gitHubOrgName: gitHubOrgName,
items: project.items.nodes.map((item: any) => ({
type: item.type,
content: {
id: item.content.id,
title: item.content.title,
url: item.content.url,
closed: item.content.closed,
labels: item.content.labels?.nodes.map((label: any) => ({
name: label.name,
})),
milestone: item.content.milestone
? {
id: item.content.milestone.id,
title: item.content.milestone.title,
description: item.content.milestone.description,
number: item.content.milestone.number,
createdAt: item.content.milestone.createdAt,
dueOn: item.content.milestone.dueOn,
state: item.content.milestone.state,
}
: null,
assignees: item.content.assignees
? item.content.assignees.nodes.map((assignee: any) => ({
id: assignee.id,
login: assignee.login,
name: assignee.name,
}))
: [],
__typename: item.content.__typename,
},
fieldValues: item.fieldValues.nodes
.filter((field: any) => field.field) // Filter out empty objects
.map((field: any) => ({
name: field.name,
field: {
name: field.field.name,
},
})),
})),
fields: project.fields.nodes
.filter((field: any) => field.options) // Filter out empty objects
.map((field: any) => ({
name: field.name,
field: {
name: field.field.name,
},
options: field.options
? field.options.map((option: any) => ({
id: option.id,
name: option.name,
}))
: [],
})),
})),
fields: project.fields.nodes
.filter((field: any) => field.options) // Filter out empty objects
.map((field: any) => ({
name: field.name,
options: field.options
? field.options.map((option: any) => ({
id: option.id,
name: option.name,
}))
: [],
})),
})
);
})
);

// Save each project into MongoDB using findOneAndUpdate
let projectId = 1;
for (const project of transformedProjects) {
try {
const jiraBoard: Omit<JiraBoard, '_id'> = {
id: projectId++,
self: project.id,
name: project.title,
type: 'GitHub Project',
jiraLocation: {
displayName: project.title,
projectName: project.title,
// Save each project into MongoDB using findOneAndUpdate
let projectId = 1;
for (const project of transformedProjects) {
try {
const jiraBoard: Omit<JiraBoard, '_id'> = {
id: projectId++,
self: project.id,
name: project.title,
},
columns: project.fields.find(
(field: { name: string }) => field.name === 'Status'
).options,
jiraIssues: [],
jiraSprints: [],
course: course._id,
};

const board = await JiraBoardModel.findOneAndUpdate(
{ self: project.id },
jiraBoard,
{
upsert: true,
new: true,
}
);

for (const item of project.items) {
const jiraIssue: Omit<JiraIssue, '_id'> = {
id: item.content.id,
self: item.content.id,
key: project.title,
storyPoints: 0,
fields: {
summary: item.content.title,
resolution: item.content.closed ? { name: 'Done' } : undefined,
issuetype: {
name: item.type,
subtask: false,
},
status: {
name:
item.fieldValues.find(
(fieldValue: { field: { name: string } }) =>
fieldValue.field.name === 'Status'
)?.name ?? 'Unknown',
},
assignee: {
displayName:
item.content.assignees.length > 0
? item.content.assignees[0].name
: null,
},
type: 'GitHub Project',
jiraLocation: {
displayName: project.title,
projectName: project.title,
name: project.title,
},
columns: project.fields.find(
(field: { name: string }) => field.name === 'Status'
).options,
jiraIssues: [],
jiraSprints: [],
course: course._id,
};

const self = jiraIssue.self;

const issue = await JiraIssueModel.findOneAndUpdate(
{ self: self },
jiraIssue,
const board = await JiraBoardModel.findOneAndUpdate(
{ self: project.id },
jiraBoard,
{
upsert: true,
new: true,
}
);

if (item.content.milestone) {
const sprint = await JiraSprintModel.findOneAndUpdate(
{ self: item.content.milestone.id },
{
$setOnInsert: {
id: item.content.milestone.number,
self: item.content.milestone.id,
name: item.content.title,
jiraIssues: [], // Initialize jiraIssues as an array
state:
item.content.milestone.state === 'CLOSED'
? 'closed'
: 'active',
startDate: item.content.milestone.createdAt,
endDate: item.content.milestone.dueOn,
createdDate: item.content.milestone.createdAt,
originBoardId: board.id,
goal: item.content.milestone.description,
for (const item of project.items) {
const jiraIssue: Omit<JiraIssue, '_id'> = {
id: item.content.id,
self: item.content.id,
key: project.title,
storyPoints: 0,
fields: {
summary: item.content.title,
resolution: item.content.closed
? { name: 'Done' }
: undefined,
issuetype: {
name: item.type,
subtask: false,
},
status: {
name:
item.fieldValues.find(
(fieldValue: { field: { name: string } }) =>
fieldValue.field.name === 'Status'
)?.name ?? 'Unknown',
},
assignee: {
displayName:
item.content.assignees.length > 0
? item.content.assignees[0].name
: null,
},
},
};

const self = jiraIssue.self;

const issue = await JiraIssueModel.findOneAndUpdate(
{ self: self },
jiraIssue,
{
upsert: true,
new: true,
}
);

// Now push the issue into the jiraIssues array
await JiraSprintModel.findOneAndUpdate(
{ self: item.content.milestone.id },
{ $addToSet: { jiraIssues: issue._id } },
{}
);
if (item.content.milestone) {
const sprint = await JiraSprintModel.findOneAndUpdate(
{ self: item.content.milestone.id },
{
$setOnInsert: {
id: item.content.milestone.number,
self: item.content.milestone.id,
name: item.content.title,
jiraIssues: [], // Initialize jiraIssues as an array
state:
item.content.milestone.state === 'CLOSED'
? 'closed'
: 'active',
startDate: item.content.milestone.createdAt,
endDate: item.content.milestone.dueOn,
createdDate: item.content.milestone.createdAt,
originBoardId: board.id,
goal: item.content.milestone.description,
},
},
{
upsert: true,
new: true,
}
);

// Now push the issue into the jiraIssues array
await JiraSprintModel.findOneAndUpdate(
{ self: item.content.milestone.id },
{ $addToSet: { jiraIssues: issue._id } },
{}
);

await JiraBoardModel.findOneAndUpdate(
{ self: project.id },
{ $addToSet: { jiraSprints: sprint._id } },
{}
);
}

await JiraBoardModel.findOneAndUpdate(
{ self: project.id },
{ $addToSet: { jiraSprints: sprint._id } },
{ $push: { jiraIssues: issue._id } },
{}
);
}

await JiraBoardModel.findOneAndUpdate(
{ self: project.id },
{ $push: { jiraIssues: issue._id } },
{}
);
console.log('Saved GitHub Project: ', project.title);
} catch (error) {
console.error('Error saving GitHub Project: ', project.id, error);
}

console.log('Saved GitHub Project: ', project.title);
} catch (error) {
console.error('Error saving GitHub Project: ', project.id, error);
}
}
}
} catch (error) {
console.error(
`Error fetching GitHub Projects for GitHub Org: ${gitHubOrgName}`
);
}
};

Expand Down
Loading