forked from breatheco-de/apiv2
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request breatheco-de#1424 from breatheco-de/development
Development
- Loading branch information
Showing
65 changed files
with
3,572 additions
and
1,860 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Large diffs are not rendered by default.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,9 +1,8 @@ | ||
| """ | ||
| For each signal you want other apps to be able to receive, you have to | ||
| declare a new variable here like this: | ||
| """ | ||
| """For each signal you want other apps to be able to receive, you have to declare a new variable here like this:""" | ||
|
|
||
| from django import dispatch | ||
| from task_manager.django.dispatch import Emisor | ||
|
|
||
| assessment_updated = dispatch.Signal() | ||
| userassessment_status_updated = dispatch.Signal() | ||
| emisor = Emisor("breathecode.assessment") | ||
|
|
||
| assessment_updated = emisor.signal("assessment_updated") | ||
| userassessment_status_updated = emisor.signal("userassessment_status_updated") |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Empty file.
Empty file.
177 changes: 177 additions & 0 deletions
177
breathecode/assignments/management/commands/schedule_repository_deletions.py
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,177 @@ | ||
| import re | ||
| from datetime import datetime | ||
| from typing import Optional | ||
|
|
||
| from dateutil import parser | ||
| from dateutil.relativedelta import relativedelta | ||
| from django.core.management.base import BaseCommand | ||
| from django.utils import timezone | ||
|
|
||
| from breathecode.assignments.models import RepositoryDeletionOrder, RepositoryWhiteList | ||
| from breathecode.authenticate.models import AcademyAuthSettings | ||
| from breathecode.monitoring.models import RepositorySubscription | ||
| from breathecode.registry.models import Asset | ||
| from breathecode.services.github import Github | ||
|
|
||
|
|
||
| class Command(BaseCommand): | ||
| help = "Clean data from marketing module" | ||
| github_url_pattern = re.compile(r"https:\/\/github\.com\/(?P<user>[^\/]+)\/(?P<repo>[^\/\s]+)\/?") | ||
|
|
||
| def handle(self, *args, **options): | ||
| self.fill_whitelist() | ||
| self.purge_deletion_orders() | ||
| self.github() | ||
|
|
||
| def github(self): | ||
| processed = set() | ||
| for settings in AcademyAuthSettings.objects.filter( | ||
| github_owner__isnull=False, github_owner__credentialsgithub__isnull=False | ||
| ).exclude(github_username=""): | ||
| self.github_client = Github( | ||
| org=settings.github_username, token=settings.github_owner.credentialsgithub.token | ||
| ) | ||
|
|
||
| key = (settings.github_username, settings.github_owner.id) | ||
| if key in processed: | ||
| continue | ||
|
|
||
| processed.add(key) | ||
| last_check = None | ||
|
|
||
| last = ( | ||
| RepositoryDeletionOrder.objects.filter(provider=RepositoryDeletionOrder.Provider.GITHUB) | ||
| .only("created_at") | ||
| .last() | ||
| ) | ||
| if last: | ||
| last_check = last.created_at | ||
|
|
||
| self.schedule_github_deletions(settings.github_username, last_check) | ||
| self.delete_github_repositories() | ||
|
|
||
| def purge_deletion_orders(self): | ||
|
|
||
| page = 0 | ||
| while True: | ||
| qs = RepositoryDeletionOrder.objects.filter( | ||
| status=RepositoryDeletionOrder.Status.PENDING, | ||
| )[page * 100 : (page + 1) * 100] | ||
|
|
||
| if len(qs) == 0: | ||
| break | ||
|
|
||
| for deletion_order in qs: | ||
| if RepositoryWhiteList.objects.filter( | ||
| provider=deletion_order.provider, | ||
| repository_user__iexact=deletion_order.repository_user, | ||
| repository_name__iexact=deletion_order.repository_name, | ||
| ).exists(): | ||
| deletion_order.delete() | ||
|
|
||
| page += 1 | ||
|
|
||
| def delete_github_repositories(self): | ||
|
|
||
| while True: | ||
| qs = RepositoryDeletionOrder.objects.filter( | ||
| provider=RepositoryDeletionOrder.Provider.GITHUB, | ||
| status=RepositoryDeletionOrder.Status.PENDING, | ||
| created_at__lte=timezone.now() - relativedelta(months=2), | ||
| )[:100] | ||
|
|
||
| if qs.count() == 0: | ||
| break | ||
|
|
||
| for deletion_order in qs: | ||
| try: | ||
| self.github_client.delete_org_repo( | ||
| owner=deletion_order.repository_user, repo=deletion_order.repository_name | ||
| ) | ||
| deletion_order.status = RepositoryDeletionOrder.Status.DELETED | ||
| deletion_order.save() | ||
|
|
||
| except Exception as e: | ||
| deletion_order.status = RepositoryDeletionOrder.Status.ERROR | ||
| deletion_order.status_text = str(e) | ||
| deletion_order.save() | ||
|
|
||
| def fill_whitelist(self): | ||
| assets = Asset.objects.filter() | ||
|
|
||
| for asset in assets: | ||
| options = [ | ||
| asset.url, | ||
| asset.solution_url, | ||
| asset.preview, | ||
| asset.readme_url, | ||
| asset.intro_video_url, | ||
| asset.solution_video_url, | ||
| ] | ||
| for url in [x for x in options if x]: | ||
| match = self.github_url_pattern.search(url) | ||
| if match: | ||
| user = match.group("user") | ||
| repo_name = match.group("repo") | ||
|
|
||
| self.add_to_whitelist("GITHUB", user, repo_name) | ||
|
|
||
| readme_raw = Asset.decode(asset.readme_raw) | ||
| if readme_raw is None: | ||
| continue | ||
|
|
||
| urls = self.github_url_pattern.findall(readme_raw) | ||
|
|
||
| for match in urls: | ||
| user, repo_name = match | ||
|
|
||
| self.add_to_whitelist("GITHUB", user, repo_name) | ||
|
|
||
| assets = Asset.objects.filter() | ||
|
|
||
| subscriptions = RepositorySubscription.objects.filter() | ||
| for subscription in subscriptions: | ||
| match = self.github_url_pattern.search(subscription.repository) | ||
| if match: | ||
| user = match.group("user") | ||
| repo_name = match.group("repo") | ||
|
|
||
| self.add_to_whitelist("GITHUB", user, repo_name) | ||
|
|
||
| def add_to_whitelist(self, provider: str, user: str, repo_name: str): | ||
| if ( | ||
| RepositoryWhiteList.objects.filter( | ||
| provider=provider, repository_user__iexact=user, repository_name__iexact=repo_name | ||
| ).exists() | ||
| is False | ||
| ): | ||
| RepositoryWhiteList.objects.get_or_create( | ||
| provider=provider, repository_user=user, repository_name=repo_name | ||
| ) | ||
|
|
||
| def schedule_github_deletions(self, organization: str, last_check: Optional[datetime] = None): | ||
| for repos in self.github_client.get_org_repos( | ||
| organization, type="forks", per_page=30, direction="desc", sort="created" | ||
| ): | ||
| for repo in repos: | ||
| created_at = parser.parse(repo["created_at"]) | ||
|
|
||
| if last_check and last_check > created_at: | ||
| return | ||
|
|
||
| if repo["fork"] is True and repo["is_template"] is False and repo["allow_forking"] is True: | ||
| match = self.github_url_pattern.search(repo["html_url"]) | ||
| if match: | ||
| user = match.group("user") | ||
| repo_name = match.group("repo") | ||
| self.schedule_github_deletion("GITHUB", user, repo_name) | ||
|
|
||
| def schedule_github_deletion(self, provider: str, user: str, repo_name: str): | ||
| if RepositoryWhiteList.objects.filter( | ||
| provider=provider, repository_user=user, repository_name=repo_name | ||
| ).exists(): | ||
| return | ||
|
|
||
| RepositoryDeletionOrder.objects.get_or_create( | ||
| provider=provider, repository_user=user, repository_name=repo_name | ||
| ) |
48 changes: 48 additions & 0 deletions
48
breathecode/assignments/migrations/0018_repositorydeletionorder_repositorywhitelist.py
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,48 @@ | ||
| # Generated by Django 5.0.6 on 2024-07-02 08:25 | ||
|
|
||
| from django.db import migrations, models | ||
|
|
||
|
|
||
| class Migration(migrations.Migration): | ||
|
|
||
| dependencies = [ | ||
| ("assignments", "0017_assignmenttelemetry_task_telemetry_learnpackwebhook"), | ||
| ] | ||
|
|
||
| operations = [ | ||
| migrations.CreateModel( | ||
| name="RepositoryDeletionOrder", | ||
| fields=[ | ||
| ("id", models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")), | ||
| ("provider", models.CharField(choices=[("GITHUB", "GitHub")], default="GITHUB", max_length=15)), | ||
| ( | ||
| "status", | ||
| models.CharField( | ||
| choices=[ | ||
| ("PENDING", "Pending"), | ||
| ("ERROR", "Error"), | ||
| ("DELETED", "Deleted"), | ||
| ("CANCELLED", "Cancelled"), | ||
| ], | ||
| default="PENDING", | ||
| max_length=15, | ||
| ), | ||
| ), | ||
| ("status_text", models.TextField(blank=True, default=None, null=True)), | ||
| ("repository_user", models.CharField(max_length=100)), | ||
| ("repository_name", models.CharField(max_length=100)), | ||
| ("created_at", models.DateTimeField(auto_now_add=True)), | ||
| ("updated_at", models.DateTimeField(auto_now=True)), | ||
| ], | ||
| ), | ||
| migrations.CreateModel( | ||
| name="RepositoryWhiteList", | ||
| fields=[ | ||
| ("id", models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")), | ||
| ("provider", models.CharField(choices=[("GITHUB", "GitHub")], default="GITHUB", max_length=15)), | ||
| ("repository_user", models.CharField(max_length=100)), | ||
| ("repository_name", models.CharField(max_length=100)), | ||
| ("created_at", models.DateTimeField(auto_now_add=True)), | ||
| ], | ||
| ), | ||
| ] |
Oops, something went wrong.