From a60ea4c2bb0c2b678889ca4f426a82df7e2e812f Mon Sep 17 00:00:00 2001
From: "kalsara.magamage_roo" <kalsara.magamage@rootcodelabs.com>
Date: Wed, 31 Jul 2024 08:34:56 +0530
Subject: [PATCH] ESCLASS-145-delete-script: implement shell script to delete
 completed data from opensearch

---
 .../DSL/dataset_progress_session.yml          |  2 +-
 ...ete_completed_dataset_progress_sessions.sh | 35 +++++++++++++++----
 ...te-completed-dataset-progress-sessions.sql |  3 +-
 3 files changed, 31 insertions(+), 9 deletions(-)

diff --git a/DSL/CronManager/DSL/dataset_progress_session.yml b/DSL/CronManager/DSL/dataset_progress_session.yml
index f11405a3..e60b8ba9 100644
--- a/DSL/CronManager/DSL/dataset_progress_session.yml
+++ b/DSL/CronManager/DSL/dataset_progress_session.yml
@@ -1,4 +1,4 @@
-token_refresh:
+delete_completed_sessions:
   trigger: "0 0 0 * * ?"
   type: exec
   command: "../app/scripts/delete_completed_dataset_progress_sessions.sh"
\ No newline at end of file
diff --git a/DSL/CronManager/script/delete_completed_dataset_progress_sessions.sh b/DSL/CronManager/script/delete_completed_dataset_progress_sessions.sh
index e32f382d..2ca3b7af 100644
--- a/DSL/CronManager/script/delete_completed_dataset_progress_sessions.sh
+++ b/DSL/CronManager/script/delete_completed_dataset_progress_sessions.sh
@@ -1,9 +1,7 @@
 #!/bin/bash
 
-# Set the working directory to the location of the script
 cd "$(dirname "$0")"
 
-# Source the constants from the ini file
 source ../config/config.ini
 
 script_name=$(basename $0)
@@ -12,15 +10,38 @@ pwd
 echo $(date -u +"%Y-%m-%d %H:%M:%S.%3NZ") - $script_name started
 
 delete_dataset_progress_sessions() {
-  delete_response=$(curl -s -X DELETE "$CLASSIFIER_RESQL/delete-completed-dataset-progress-sessions")
-  if echo "$delete_response" | grep -q '"success":true'; then
-    echo "Data deletion successful"
+  delete_response=$(curl -s -X POST -H "Content-Type: application/json" "http://resql:8082/delete-completed-dataset-progress-sessions")
+
+  echo "Response from delete request: $delete_response"
+
+  session_ids=$(echo "$delete_response" | grep -oP '"id":\K\d+' | tr '\n' ' ' | sed 's/ $//')  # Remove trailing space
+
+  echo "Session IDs: $session_ids"
+
+  if [ -n "$session_ids" ]; then
+    delete_from_opensearch "$session_ids"
   else
-    echo "Data deletion failed: $delete_response"
-    exit 1
+    echo "No session IDs were returned in the response."
   fi
 }
 
+delete_from_opensearch() {
+  local session_ids="$1"
+
+  delete_query="{\"query\": {\"terms\": {\"sessionId\": ["
+  for id in $session_ids; do
+    delete_query+="\"$id\","
+  done
+  delete_query=$(echo "$delete_query" | sed 's/,$//') # Remove trailing comma
+  delete_query+="]}}}"
+
+  echo "delete query: $delete_query"
+
+  opensearch_response=$(curl -s -X POST -H "Content-Type: application/json" -d "$delete_query" "http://opensearch-node:9200/dataset_progress_sessions/_delete_by_query")
+
+  echo "Response from OpenSearch delete request: $opensearch_response"
+}
+
 delete_dataset_progress_sessions
 
 echo $(date -u +"%Y-%m-%d %H:%M:%S.%3NZ") - $script_name finished
diff --git a/DSL/Resql/delete-completed-dataset-progress-sessions.sql b/DSL/Resql/delete-completed-dataset-progress-sessions.sql
index 4ea006b9..eb0b5728 100644
--- a/DSL/Resql/delete-completed-dataset-progress-sessions.sql
+++ b/DSL/Resql/delete-completed-dataset-progress-sessions.sql
@@ -1,2 +1,3 @@
 DELETE FROM dataset_progress_sessions
-WHERE process_complete = true;
+WHERE process_complete = true
+RETURNING id;