Merge pull request #129 from rootcodelabs/dev

Getting update from dev to 132

DSL/CronManager/DSL/dataset_progress_session.yml

@@ -1,4 +1,4 @@
-token_refresh:
+delete_completed_sessions:
   trigger: "0 0 0 * * ?"
   type: exec
   command: "../app/scripts/delete_completed_dataset_progress_sessions.sh"

DSL/CronManager/script/delete_completed_dataset_progress_sessions.sh

@@ -1,9 +1,7 @@
 #!/bin/bash
 
-# Set the working directory to the location of the script
 cd "$(dirname "$0")"
 
-# Source the constants from the ini file
 source ../config/config.ini
 
 script_name=$(basename $0)
@@ -12,15 +10,38 @@ pwd
 echo $(date -u +"%Y-%m-%d %H:%M:%S.%3NZ") - $script_name started
 
 delete_dataset_progress_sessions() {
-  delete_response=$(curl -s -X DELETE "$CLASSIFIER_RESQL/delete-completed-dataset-progress-sessions")
-  if echo "$delete_response" | grep -q '"success":true'; then
-    echo "Data deletion successful"
+  delete_response=$(curl -s -X POST -H "Content-Type: application/json" "http://resql:8082/delete-completed-dataset-progress-sessions")
+
+  echo "Response from delete request: $delete_response"
+
+  session_ids=$(echo "$delete_response" | grep -oP '"id":\K\d+' | tr '\n' ' ' | sed 's/ $//')  # Remove trailing space
+
+  echo "Session IDs: $session_ids"
+
+  if [ -n "$session_ids" ]; then
+    delete_from_opensearch "$session_ids"
   else
-    echo "Data deletion failed: $delete_response"
-    exit 1
+    echo "No session IDs were returned in the response."
   fi
 }
 
+delete_from_opensearch() {
+  local session_ids="$1"
+
+  delete_query="{\"query\": {\"terms\": {\"sessionId\": ["
+  for id in $session_ids; do
+    delete_query+="\"$id\","
+  done
+  delete_query=$(echo "$delete_query" | sed 's/,$//') # Remove trailing comma
+  delete_query+="]}}}"
+
+  echo "delete query: $delete_query"
+
+  opensearch_response=$(curl -s -X POST -H "Content-Type: application/json" -d "$delete_query" "http://opensearch-node:9200/dataset_progress_sessions/_delete_by_query")
+
+  echo "Response from OpenSearch delete request: $opensearch_response"
+}
+
 delete_dataset_progress_sessions
 
 echo $(date -u +"%Y-%m-%d %H:%M:%S.%3NZ") - $script_name finished

DSL/Resql/delete-completed-dataset-progress-sessions.sql

@@ -1,2 +1,3 @@
 DELETE FROM dataset_progress_sessions
-WHERE process_complete = true;
+WHERE process_complete = true
+RETURNING id;