From 81a8780b9419befb3b7938a5e0e06ce76c8ec892 Mon Sep 17 00:00:00 2001 From: Omar Yasser Date: Sat, 11 May 2024 11:23:24 +0100 Subject: [PATCH] Clean empty group IDs (#464) --- deploy-jobs.sh | 23 ++++++ .../com/azkar/crons/ChallengesCleaner.java | 2 +- .../crons/FriendshipEmptyGroupIdCleaner.java | 82 +++++++++++++++++++ src/main/resources/application.yml | 4 +- tanafaso-cloud-run-jobs.yaml | 21 +++++ tanafaso-cloud-run-service.yaml | 2 +- 6 files changed, 131 insertions(+), 3 deletions(-) create mode 100755 deploy-jobs.sh create mode 100644 src/main/java/com/azkar/crons/FriendshipEmptyGroupIdCleaner.java create mode 100644 tanafaso-cloud-run-jobs.yaml diff --git a/deploy-jobs.sh b/deploy-jobs.sh new file mode 100755 index 0000000..62f4566 --- /dev/null +++ b/deploy-jobs.sh @@ -0,0 +1,23 @@ +#!/bin/bash + +# Note: Make sure that the desired jobs are marked true in application.yaml + +# Exit script when a command fails +set -e + +./mvnw package -Dmaven.test.skip=true + +docker build --platform linux/amd64 -t tanafaso . + +random_tag=$(printf "randomtag%08d" $((RANDOM%100000000))) + +docker tag tanafaso europe-west1-docker.pkg.dev/tanafaso/tanafaso-jobs/tanafaso-jobs:$random_tag + +docker push europe-west1-docker.pkg.dev/tanafaso/tanafaso-jobs/tanafaso-jobs:$random_tag + +sed -i '' "s|image: europe-west1-docker.pkg.dev/tanafaso/tanafaso-jobs/tanafaso-jobs:.*|image: europe-west1-docker.pkg.dev/tanafaso/tanafaso-jobs/tanafaso-jobs:$random_tag|" tanafaso-cloud-run-jobs.yaml + +gcloud run jobs replace tanafaso-cloud-run-jobs.yaml + +echo "Deployed with tag: $random_tag" + diff --git a/src/main/java/com/azkar/crons/ChallengesCleaner.java b/src/main/java/com/azkar/crons/ChallengesCleaner.java index f8ca425..49bcb36 100644 --- a/src/main/java/com/azkar/crons/ChallengesCleaner.java +++ b/src/main/java/com/azkar/crons/ChallengesCleaner.java @@ -29,7 +29,7 @@ public class ChallengesCleaner implements ApplicationRunner { @Autowired private ApplicationContext appContext; - @Value("${job-mode}") + @Value("${challenges-cleaner-job-run-mode}") public boolean jobMode; // Run every while to clean old challenges. Note that although after every challenge creation diff --git a/src/main/java/com/azkar/crons/FriendshipEmptyGroupIdCleaner.java b/src/main/java/com/azkar/crons/FriendshipEmptyGroupIdCleaner.java new file mode 100644 index 0000000..cae48ce --- /dev/null +++ b/src/main/java/com/azkar/crons/FriendshipEmptyGroupIdCleaner.java @@ -0,0 +1,82 @@ +package com.azkar.crons; + +import com.azkar.entities.Friendship; +import com.azkar.entities.Friendship.Friend; +import com.azkar.repos.FriendshipRepo; +import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.stream.Collectors; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.ApplicationArguments; +import org.springframework.boot.ApplicationRunner; +import org.springframework.boot.SpringApplication; +import org.springframework.context.ApplicationContext; +import org.springframework.data.domain.Page; +import org.springframework.data.domain.PageRequest; +import org.springframework.stereotype.Component; + +@Component +public class FriendshipEmptyGroupIdCleaner implements ApplicationRunner { + + private static final Logger logger = LoggerFactory.getLogger(FriendshipEmptyGroupIdCleaner.class); + private static final int FRIENDSHIPS_BATCH_SIZE = 100; + @Value("${friendship-empty-group-id-cleaner-job-run-mode}") + public boolean jobMode; + @Autowired + private FriendshipRepo friendshipRepo; + @Autowired + private ApplicationContext appContext; + + // This is a cleanup after the fix https://github.com/tanafaso/tanafaso-backend/pull/463. + @Override + public void run(ApplicationArguments args) throws Exception { + if (!jobMode) { + logger.info( + "[Empty group ID cleaner] skipping as the application is not running in job mode"); + return; + } + + logger.info("[Empty group ID cleaner] started!"); + + long numberOfFriendships = friendshipRepo.count(); + logger.info("[Empty group ID cleaner] number of friendships to be processed: {}", + numberOfFriendships); + logger.info("[Empty group ID cleaner] friendships will be processed in batches of {}", + FRIENDSHIPS_BATCH_SIZE); + + long numberOfBatches = + (numberOfFriendships + FRIENDSHIPS_BATCH_SIZE - 1) / FRIENDSHIPS_BATCH_SIZE; + logger.info("[Empty group ID cleaner] number of batches to be processed: {}", numberOfBatches); + + AtomicInteger cleanedFriendshps = new AtomicInteger(); + for (int batch = 0; batch < numberOfBatches; batch++) { + Page page = friendshipRepo.findAll(PageRequest.of(batch, FRIENDSHIPS_BATCH_SIZE)); + + List modifiedFriendships = page.get().map(friendship -> { + int friendsCountBeforeCleaning = friendship.getFriends().size(); + + List cleanedFriendships = + friendship.getFriends().stream().filter(friend -> friend.getGroupId() != null) + .collect(Collectors.toList()); + + int friendsCountAfterCleaning = cleanedFriendships.size(); + int cleanedFriendshipsCount = friendsCountBeforeCleaning - friendsCountAfterCleaning; + if (cleanedFriendshipsCount != 0) { + friendship.setFriends(cleanedFriendships); + logger.info("[Empty group ID cleaner] cleaned {} friendships for user {}", + cleanedFriendshipsCount, friendship.getUserId()); + } + cleanedFriendshps.addAndGet((cleanedFriendshipsCount)); + return friendship; + }).collect(Collectors.toList()); + + friendshipRepo.saveAll(modifiedFriendships); + + logger.info("[Empty group ID cleaner] processed {}/{} batches", batch + 1, numberOfBatches); + } + SpringApplication.exit(appContext, () -> 0); + } +} \ No newline at end of file diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml index e20b646..ee88b74 100644 --- a/src/main/resources/application.yml +++ b/src/main/resources/application.yml @@ -30,4 +30,6 @@ files: quran-metadata: quran_metadata.csv apple_auth_private_key: sign_in_with_apple_auth_key -job-mode: false +challenges-cleaner-job-run-mode: false +friendship-empty-group-id-cleaner-job-run-mode: true + diff --git a/tanafaso-cloud-run-jobs.yaml b/tanafaso-cloud-run-jobs.yaml new file mode 100644 index 0000000..f3bbb8c --- /dev/null +++ b/tanafaso-cloud-run-jobs.yaml @@ -0,0 +1,21 @@ +apiVersion: run.googleapis.com/v1 +kind: Job +metadata: + name: tanafaso-jobs + namespace: 'tanafaso' + labels: + cloud.googleapis.com/location: europe-west1 + annotations: + run.googleapis.com/ingress: all + run.googleapis.com/ingress-status: all +spec: + template: + spec: + parallelism: 1 + taskCount: 1 + template: + spec: + maxRetries: 0 + timeoutSeconds: 3600 + containers: + - image: europe-west1-docker.pkg.dev/tanafaso/tanafaso-jobs/tanafaso-jobs:randomtag00027311 diff --git a/tanafaso-cloud-run-service.yaml b/tanafaso-cloud-run-service.yaml index 4200010..2f5c4d1 100644 --- a/tanafaso-cloud-run-service.yaml +++ b/tanafaso-cloud-run-service.yaml @@ -22,7 +22,7 @@ spec: timeoutSeconds: 15 containers: - name: tanafaso-1 - image: europe-west1-docker.pkg.dev/tanafaso/tanafaso/tanafaso:randomtag00010272 + image: europe-west1-docker.pkg.dev/tanafaso/tanafaso/tanafaso:randomtag00020031 ports: - name: http1 containerPort: 443