Merge branch 'comments' of https://github.com/krrish-sehgal/Owasp-BLT …

…into comments

.github/workflows/ci-cd.yml

@@ -155,7 +155,7 @@ jobs:
         with:
           python-version: 3.11.2
       - run: pip install poetry
-      - run: poetry lock --no-update
+      - run: poetry lock
       - run: poetry install
       - run: poetry run python manage.py collectstatic --noinput
       - name: Run tests
@@ -186,8 +186,8 @@ jobs:
           docker run -d --name my-container my-app
 
       - run: docker exec my-container pip install poetry
-      - run: docker exec my-container poetry lock --no-update
-      - run: docker exec my-container poetry install --no-dev --no-interaction
+      - run: docker exec my-container poetry lock
+      - run: docker exec my-container poetry install --without dev --no-interaction
 
       - name: Clean up
         run: |

Dockerfile

@@ -33,8 +33,8 @@ RUN ln -s /usr/bin/google-chrome-stable /usr/local/bin/google-chrome
 RUN pip install poetry
 RUN poetry config virtualenvs.create false
 COPY pyproject.toml poetry.lock* ./
-RUN poetry lock --no-update
-RUN poetry install
+RUN poetry lock
+RUN poetry install --no-root
 
 # Install additional Python packages
 RUN pip install opentelemetry-api opentelemetry-instrumentation

pyproject.toml

@@ -65,7 +65,7 @@ sentry-sdk = "^2.19.0"
 bitcash = "^1.0.2"
 pydantic = "^2.7.3"
 pydantic_core = "^2.18.4"
-unstructured = "^0.16.8"
+unstructured = "^0.16.12"
 Markdown = "^3.6"
 faiss-cpu = "^1.8.0"
 psutil = "^5.9.8"
@@ -87,7 +87,7 @@ newrelic = "^10.4.0"
 [tool.poetry.group.dev.dependencies]
 black = "^24.8.0"
 isort = "^5.13.2"
-ruff = "^0.8.4"
+ruff = "^0.8.6"
 pre-commit = "^3.8.0"
 
 [tool.isort]

website/management/commands/sync_repo_contributors.py

@@ -0,0 +1,82 @@
+import logging
+import time
+
+import requests
+from django.conf import settings
+from django.core.management.base import BaseCommand
+
+from website.models import Contributor, Repo
+
+logger = logging.getLogger(__name__)
+
+
+class Command(BaseCommand):
+    help = "Synchronize all contributors for a repository"
+
+    def add_arguments(self, parser):
+        parser.add_argument("--repo_id", type=int, help="Repository ID to sync")
+
+    def handle(self, *args, **options):
+        repo_id = options.get("repo_id")
+        if not repo_id:
+            return
+
+        repo = Repo.objects.get(id=repo_id)
+        owner_repo = repo.repo_url.rstrip("/").split("github.com/")[-1]
+
+        headers = {
+            "Authorization": f"token {settings.GITHUB_TOKEN}",
+            "Accept": "application/vnd.github.v3+json",
+        }
+
+        # Get all contributors with pagination
+        page = 1
+        all_contributors = []
+
+        while True:
+            api_url = f"https://api.github.com/repos/{owner_repo}/contributors?anon=true&per_page=100&page={page}"
+            response = requests.get(api_url, headers=headers)
+
+            if response.status_code == 403:
+                reset_time = int(response.headers.get("X-RateLimit-Reset", 0))
+                wait_time = reset_time - int(time.time())
+                if wait_time > 0:
+                    logger.info(f"Rate limit hit, waiting {wait_time} seconds")
+                    time.sleep(wait_time)
+                    continue
+
+            if response.status_code != 200:
+                break
+
+            contributors_page = response.json()
+            if not contributors_page:
+                break
+
+            all_contributors.extend(contributors_page)
+            page += 1
+
+            # Be nice to GitHub API
+            time.sleep(1)
+
+        # Batch create/update contributors
+        for contrib_data in all_contributors:
+            github_id = contrib_data.get("id")
+            if not github_id:
+                # skip if 'id' is missing
+                continue
+            contributor, created = Contributor.objects.update_or_create(
+                github_id=github_id,
+                defaults={
+                    "name": contrib_data.get("login", "unknown"),
+                    "github_url": contrib_data.get("html_url", ""),
+                    "avatar_url": contrib_data.get("avatar_url", ""),
+                    "contributions": contrib_data.get("contributions", 0),
+                    "contributor_type": contrib_data.get("type", "User"),
+                },
+            )
+            repo.contributor.add(contributor)
+
+        repo.contributor_count = len(all_contributors)
+        repo.save()
+
+        logger.info(f"Synced {len(all_contributors)} contributors for {repo.name}")

website/migrations/0179_contributorstats.py

@@ -0,0 +1,60 @@
+# Generated by Django 5.1.3 on 2025-01-06 13:17
+
+import django.db.models.deletion
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("website", "0178_alter_ip_agent"),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name="ContributorStats",
+            fields=[
+                (
+                    "id",
+                    models.AutoField(
+                        auto_created=True,
+                        primary_key=True,
+                        serialize=False,
+                        verbose_name="ID",
+                    ),
+                ),
+                ("date", models.DateField()),
+                ("commits", models.PositiveIntegerField(default=0)),
+                ("issues_opened", models.PositiveIntegerField(default=0)),
+                ("issues_closed", models.PositiveIntegerField(default=0)),
+                ("pull_requests", models.PositiveIntegerField(default=0)),
+                ("comments", models.PositiveIntegerField(default=0)),
+                (
+                    "granularity",
+                    models.CharField(
+                        choices=[("day", "Day"), ("month", "Month")],
+                        default="day",
+                        max_length=10,
+                    ),
+                ),
+                (
+                    "contributor",
+                    models.ForeignKey(
+                        on_delete=django.db.models.deletion.CASCADE,
+                        related_name="stats",
+                        to="website.contributor",
+                    ),
+                ),
+                (
+                    "repo",
+                    models.ForeignKey(
+                        on_delete=django.db.models.deletion.CASCADE,
+                        related_name="stats",
+                        to="website.repo",
+                    ),
+                ),
+            ],
+            options={
+                "unique_together": {("contributor", "repo", "date", "granularity")},
+            },
+        ),
+    ]

website/models.py

@@ -936,17 +936,6 @@ def __str__(self):
         return self.name
 
 
-# class ContributorStats(models.Model):
-#     username = models.CharField(max_length=255, unique=True)
-#     commits = models.IntegerField(default=0)
-#     issues_opened = models.IntegerField(default=0)
-#     issues_closed = models.IntegerField(default=0)
-#     prs = models.IntegerField(default=0)
-#     comments = models.IntegerField(default=0)
-#     assigned_issues = models.IntegerField(default=0)
-#     created = models.DateTimeField(auto_now_add=True)
-
-
 class Contribution(models.Model):
     CONTRIBUTION_TYPES = [
         ("commit", "Commit"),
@@ -1318,3 +1307,32 @@ def save(self, *args, **kwargs):
 
     def __str__(self):
         return f"{self.project.name}/{self.name}"
+
+
+class ContributorStats(models.Model):
+    contributor = models.ForeignKey(Contributor, on_delete=models.CASCADE, related_name="stats")
+    repo = models.ForeignKey(Repo, on_delete=models.CASCADE, related_name="stats")
+
+    # This will represent either a specific day or the first day of a month.
+    date = models.DateField()
+
+    # Store counts
+    commits = models.PositiveIntegerField(default=0)
+    issues_opened = models.PositiveIntegerField(default=0)
+    issues_closed = models.PositiveIntegerField(default=0)
+    pull_requests = models.PositiveIntegerField(default=0)
+    comments = models.PositiveIntegerField(default=0)
+
+    # "day" for daily entries, "month" for monthly entries
+    granularity = models.CharField(
+        max_length=10, choices=[("day", "Day"), ("month", "Month")], default="day"
+    )
+
+    class Meta:
+        # You can't have two different stats for the same date+granularity
+        unique_together = ("contributor", "repo", "date", "granularity")
+
+    def __str__(self):
+        return (
+            f"{self.contributor.name} in {self.repo.name} " f"on {self.date} [{self.granularity}]"
+        )