feat: add GitHub issue ingestion with admin

backend/Cargo.toml

@@ -36,11 +36,11 @@ anyhow = "1.0"
 chrono = { version = "0.4", features = ["serde"] }
 uuid = { version = "1.0", features = ["v4", "serde"] }
 regex = "1.0"
+reqwest = { version = "0.12", features = ["json", "rustls-tls"], default-features = false }
 
 # Environment
 dotenvy = "0.15"
 
 [dev-dependencies]
 tokio = { version = "1.0", features = ["macros"] }
-reqwest = { version = "0.11", features = ["json"] }
 hyper = { version = "0.14", features = ["full"] }

backend/README.md

@@ -411,3 +411,93 @@ docker run -e DATABASE_URL=postgresql://... guild-backend
 - `src/application`: commands, queries, and DTOs
 - `migrations/`: SQLx migration files
 - `.sqlx/`: SQLx offline query metadata (committed to repo)
+
+## 12) GitHub Issue Ingestion
+
+The backend can sync GitHub issues into the database via an admin-protected endpoint.
+
+### Required Environment Variables
+
+Add these to `backend/.env`:
+
+```
+GITHUB_TOKEN=ghp_your_personal_access_token
+GITHUB_OWNER=TheSoftwareDevGuild
+GITHUB_API_URL=https://api.github.com
+```
+
+| Variable | Required | Description |
+|---|---|---|
+| `GITHUB_TOKEN` | Yes | GitHub personal access token (PAT) with `repo` scope |
+| `GITHUB_OWNER` | Yes | GitHub organization or user that owns the repos |
+| `GITHUB_API_URL` | No | API base URL (defaults to `https://api.github.com`) |
+
+### Trigger Sync (Admin)
+
+The sync endpoint is protected by admin authentication. You need a wallet address listed in the `ADMIN_ADDRESSES` environment variable.
+
+```bash
+# Sync issues for one or more repositories
+curl -X POST http://localhost:3001/admin/github/sync \
+  -H "Content-Type: application/json" \
+  -H "x-eth-address: <YOUR_ADMIN_ADDRESS>" \
+  -d '{
+    "repos": ["TheGuildGenesis"],
+    "since": "2025-01-01T00:00:00Z"
+  }'
+```
+
+**Request body**:
+- `repos` (required): List of repository names under `GITHUB_OWNER` to sync
+- `since` (optional): ISO 8601 timestamp — only sync issues updated after this date
+
+**Response**:
+```json
+{
+  "synced": 42,
+  "repos": ["TheGuildGenesis"]
+}
+```
+
+### How It Works
+- Fetches issues via `{GITHUB_API_URL}/repos/{GITHUB_OWNER}/{repo}/issues`
+- Ignores pull requests (GitHub returns PRs in the issues endpoint)
+- Derives `points` from labels matching the pattern `Npts` (e.g. `3pts`, `10pts`, case-insensitive)
+- Normalizes all labels to lowercase
+- Upserts using composite key `(repo_id, github_issue_id)` for idempotency
+- Preserves `rewarded_sepolia` and `distribution_id` across re-syncs
+
+### Fetch Synced Issues (Public)
+
+After syncing, query the stored issues to verify:
+
+```bash
+# List all synced issues for a repo
+curl http://localhost:3001/github/issues?repo=TheGuildGenesis
+
+# Filter by state
+curl "http://localhost:3001/github/issues?repo=TheGuildGenesis&state=closed"
+```
+
+**Response** (array of `GithubIssue`):
+```json
+[
+  {
+    "repo_id": 123456,
+    "github_issue_id": 789,
+    "repo": "TheGuildGenesis",
+    "issue_number": 42,
+    "title": "Implement feature X",
+    "state": "open",
+    "labels": ["bug", "3pts"],
+    "points": 3,
+    "assignee_logins": ["alice"],
+    "url": "https://github.com/TheSoftwareDevGuild/TheGuildGenesis/issues/42",
+    "created_at": "2025-01-15T10:00:00Z",
+    "closed_at": null,
+    "rewarded_sepolia": false,
+    "distribution_id": null,
+    "updated_at": "2025-01-20T12:00:00Z"
+  }
+]
+```

backend/migrations/006_create_github_issues_table.sql

@@ -0,0 +1,22 @@
+CREATE TABLE IF NOT EXISTS github_issues (
+    repo_id BIGINT NOT NULL,
+    github_issue_id BIGINT NOT NULL,
+    repo TEXT NOT NULL,
+    issue_number INT NOT NULL,
+    title TEXT NOT NULL,
+    state TEXT NOT NULL CHECK (state IN ('open', 'closed')),
+    labels JSONB NOT NULL DEFAULT '[]',
+    points INT NOT NULL DEFAULT 0,
+    assignee_logins JSONB NOT NULL DEFAULT '[]',
+    url TEXT NOT NULL,
+    created_at TIMESTAMPTZ NOT NULL,
+    closed_at TIMESTAMPTZ,
+    rewarded_sepolia BOOLEAN NOT NULL DEFAULT false,
+    distribution_id TEXT,
+    updated_at TIMESTAMPTZ NOT NULL,
+    PRIMARY KEY (repo_id, github_issue_id)
+);
+
+CREATE INDEX IF NOT EXISTS idx_github_issues_repo ON github_issues(repo);
+CREATE INDEX IF NOT EXISTS idx_github_issues_state ON github_issues(state);
+CREATE INDEX IF NOT EXISTS idx_github_issues_rewarded_sepolia ON github_issues(rewarded_sepolia);

backend/src/application/commands/mod.rs

@@ -2,5 +2,6 @@ pub mod create_profile;
 pub mod create_project;
 pub mod delete_project;
 pub mod login;
+pub mod sync_github_issues;
 pub mod update_profile;
 pub mod update_project;

backend/src/application/commands/sync_github_issues.rs

@@ -0,0 +1,113 @@
+use std::sync::Arc;
+
+use regex::Regex;
+
+use crate::domain::{
+    entities::github_issue::GithubIssue,
+    repositories::github_issue_repository::GithubIssueRepository,
+    services::github_service::{GitHubApiIssue, GithubService},
+};
+
+/// Derive points from labels matching the pattern `Npts` (e.g. `3pts`, `10pts`).
+/// Label names are normalized to lower-case.
+pub fn derive_points(labels: &[crate::domain::services::github_service::GitHubApiLabel]) -> i32 {
+    let re = Regex::new(r"^(\d+)pts$").expect("Invalid regex");
+    for label in labels {
+        let name = label.name.to_lowercase();
+        if let Some(caps) = re.captures(&name) {
+            if let Ok(pts) = caps[1].parse::<i32>() {
+                return pts;
+            }
+        }
+    }
+    0
+}
+
+/// Transform a GitHub API issue into a domain GithubIssue entity.
+pub fn transform_issue(
+    repo: &str,
+    repo_id: i64,
+    api_issue: &GitHubApiIssue,
+) -> Result<GithubIssue, String> {
+    let labels_normalized: Vec<serde_json::Value> = api_issue
+        .labels
+        .iter()
+        .map(|l| serde_json::Value::String(l.name.to_lowercase()))
+        .collect();
+
+    let assignee_logins: Vec<serde_json::Value> = api_issue
+        .assignees
+        .iter()
+        .map(|a| serde_json::Value::String(a.login.clone()))
+        .collect();
+
+    let points = derive_points(&api_issue.labels);
+
+    let created_at = chrono::DateTime::parse_from_rfc3339(&api_issue.created_at)
+        .map_err(|e| format!("Invalid created_at: {e}"))?
+        .with_timezone(&chrono::Utc);
+
+    let closed_at = api_issue
+        .closed_at
+        .as_ref()
+        .map(|s| chrono::DateTime::parse_from_rfc3339(s).map(|dt| dt.with_timezone(&chrono::Utc)))
+        .transpose()
+        .map_err(|e| format!("Invalid closed_at: {e}"))?;
+
+    let updated_at = chrono::DateTime::parse_from_rfc3339(&api_issue.updated_at)
+        .map_err(|e| format!("Invalid updated_at: {e}"))?
+        .with_timezone(&chrono::Utc);
+
+    Ok(GithubIssue {
+        repo_id,
+        github_issue_id: api_issue.id,
+        repo: repo.to_string(),
+        issue_number: api_issue.number,
+        title: api_issue.title.clone(),
+        state: api_issue.state.clone(),
+        labels: serde_json::Value::Array(labels_normalized),
+        points,
+        assignee_logins: serde_json::Value::Array(assignee_logins),
+        url: api_issue.html_url.clone(),
+        created_at,
+        closed_at,
+        rewarded_sepolia: false,
+        distribution_id: None,
+        updated_at,
+    })
+}
+
+/// Sync GitHub issues for the given repos.
+pub async fn sync_github_issues(
+    github_service: Arc<dyn GithubService>,
+    issue_repository: Arc<dyn GithubIssueRepository>,
+    repos: Vec<String>,
+    since: Option<String>,
+) -> Result<usize, String> {
+    let mut total_synced: usize = 0;
+
+    for repo in &repos {
+        let (repo_id, api_issues) = github_service
+            .fetch_issues(repo, since.as_deref())
+            .await
+            .map_err(|e| format!("Failed to fetch issues for {repo}: {e}"))?;
+
+        for api_issue in &api_issues {
+            // Ignore PRs
+            if api_issue.pull_request.is_some() {
+                continue;
+            }
+
+            let issue = transform_issue(repo, repo_id, api_issue)?;
+
+            issue_repository
+                .upsert(&issue)
+                .await
+                .map_err(|e| format!("Failed to upsert issue {}: {e}", api_issue.id))?;
+
+            total_synced += 1;
+        }
+    }
+
+    Ok(total_synced)
+}

backend/src/application/dtos/github_dtos.rs

@@ -0,0 +1,22 @@
+use serde::{Deserialize, Serialize};
+
+/// Request DTO for POST /admin/github/sync
+#[derive(Debug, Deserialize)]
+pub struct GithubSyncRequest {
+    pub repos: Vec<String>,
+    pub since: Option<String>,
+}
+
+/// Response DTO for POST /admin/github/sync
+#[derive(Debug, Serialize)]
+pub struct GithubSyncResponse {
+    pub synced: usize,
+    pub repos: Vec<String>,
+}
+
+/// Query parameters for GET /github/issues
+#[derive(Debug, Deserialize)]
+pub struct GithubIssuesQuery {
+    pub repo: String,
+    pub state: Option<String>,
+}

backend/src/application/dtos/mod.rs

@@ -1,4 +1,5 @@
 pub mod auth_dtos;
+pub mod github_dtos;
 pub mod profile_dtos;
 pub mod project_dtos;
 pub use auth_dtos::*;

backend/src/domain/entities/github_issue.rs

@@ -0,0 +1,21 @@
+use chrono::{DateTime, Utc};
+use serde::{Deserialize, Serialize};
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct GithubIssue {
+    pub repo_id: i64,
+    pub github_issue_id: i64,
+    pub repo: String,
+    pub issue_number: i32,
+    pub title: String,
+    pub state: String,
+    pub labels: serde_json::Value,
+    pub points: i32,
+    pub assignee_logins: serde_json::Value,
+    pub url: String,
+    pub created_at: DateTime<Utc>,
+    pub closed_at: Option<DateTime<Utc>>,
+    pub rewarded_sepolia: bool,
+    pub distribution_id: Option<String>,
+    pub updated_at: DateTime<Utc>,
+}

backend/src/domain/entities/mod.rs

@@ -1,3 +1,4 @@
+pub mod github_issue;
 pub mod profile;
 pub mod projects;
 

backend/src/domain/repositories/github_issue_repository.rs

@@ -0,0 +1,23 @@
+use async_trait::async_trait;
+
+use crate::domain::entities::github_issue::GithubIssue;
+
+#[async_trait]
+pub trait GithubIssueRepository: Send + Sync {
+    /// Upsert a GitHub issue (insert or update based on composite key repo_id + github_issue_id)
+    async fn upsert(&self, issue: &GithubIssue) -> Result<(), Box<dyn std::error::Error>>;
+
+    /// Find an issue by its composite key
+    async fn find_by_key(
+        &self,
+        repo_id: i64,
+        github_issue_id: i64,
+    ) -> Result<Option<GithubIssue>, Box<dyn std::error::Error>>;
+
+    /// List issues filtered by repo name and optional state
+    async fn list_by_repo(
+        &self,
+        repo: &str,
+        state: Option<&str>,
+    ) -> Result<Vec<GithubIssue>, Box<dyn std::error::Error>>;
+}

backend/src/domain/repositories/mod.rs

@@ -1,5 +1,7 @@
+pub mod github_issue_repository;
 pub mod profile_repository;
 pub mod project_repository;
 
+pub use github_issue_repository::GithubIssueRepository;
 pub use profile_repository::ProfileRepository;
 pub use project_repository::ProjectRepository;

backend/src/domain/services/github_service.rs

@@ -0,0 +1,45 @@
+use async_trait::async_trait;
+use serde::Deserialize;
+
+/// Raw issue data returned from the GitHub API
+#[derive(Debug, Clone, Deserialize)]
+pub struct GitHubApiIssue {
+    pub id: i64,
+    pub number: i32,
+    pub title: String,
+    pub state: String,
+    pub html_url: String,
+    pub labels: Vec<GitHubApiLabel>,
+    pub assignees: Vec<GitHubApiUser>,
+    pub created_at: String,
+    pub closed_at: Option<String>,
+    pub updated_at: String,
+    pub pull_request: Option<serde_json::Value>,
+}
+
+#[derive(Debug, Clone, Deserialize)]
+pub struct GitHubApiLabel {
+    pub name: String,
+}
+
+#[derive(Debug, Clone, Deserialize)]
+pub struct GitHubApiUser {
+    pub login: String,
+}
+
+#[derive(Debug, Clone, Deserialize)]
+pub struct GitHubApiRepo {
+    pub id: i64,
+}
+
+#[async_trait]
+pub trait GithubService: Send + Sync {
+    /// Fetch issues from a GitHub repository via REST API.
+    /// `repo` is the repository name (e.g. "TheGuildGenesis"); owner comes from GITHUB_OWNER env var.
+    /// `since` is an optional ISO 8601 timestamp to filter issues updated since that time.
+    async fn fetch_issues(
+        &self,
+        repo: &str,
+        since: Option<&str>,
+    ) -> Result<(i64, Vec<GitHubApiIssue>), Box<dyn std::error::Error>>;
+}

backend/src/domain/services/mod.rs

@@ -1 +1,2 @@
 pub mod auth_service;
+pub mod github_service;

backend/src/infrastructure/repositories/mod.rs

@@ -1,5 +1,7 @@
+pub mod postgres_github_issue_repository;
 pub mod postgres_profile_repository;
 pub mod postgres_project_repository;
 
+pub use postgres_github_issue_repository::PostgresGithubIssueRepository;
 pub use postgres_profile_repository::PostgresProfileRepository;
 pub use postgres_project_repository::PostgresProjectRepository;