Fix/loki backups (#23)

* fix: duplicate logs issues between loki upgrades

* rollback loki version

* fixes

* fixes

* fixes

* fixes

* bump loki version

---------

Co-authored-by: Venkata Mutyala <venkata.mutytala@glueops.dev>

loki-logcli-backup.sh

@@ -42,39 +42,50 @@ for i in {2..72}; do
 
   # Prepare part file name
   prefix_file_name="loki_v${LOGCLI_VERSION//./-}__"
-  part_file="$prefix_file_name$(date -u -d "$start_time" '+%Y%m%dT%H%M%S')_$(date -u -d "$end_time" '+%Y%m%dT%H%M%S').part"
+  time_window_of_logs="$(date -u -d "$start_time" '+%Y%m%dT%H%M%S')_$(date -u -d "$end_time" '+%Y%m%dT%H%M%S').part"
+  part_file="${prefix_file_name}${time_window_of_logs}"
   echo "part_file: $part_file"
 
   # Prepare S3 path
-  s3_path="${s3_key_prefix}/$(date -u -d "$start_time" '+%Y/%m/%d/%H')/${part_file}.gz"
+  S3_FOLDER_PATH="${s3_key_prefix}/$(date -u -d "$start_time" '+%Y/%m/%d/%H')/"
+  s3_path="${S3_FOLDER_PATH}${part_file}.gz"
   echo "s3_path: $s3_path"
 
   # Check if the file already exists in S3 and has been replicated.
-  STATUS=$(aws s3api head-object --bucket "$S3_BUCKET_NAME" --key "${s3_path}" | jq .ReplicationStatus -r) || true
-  if [[ "$STATUS" == "COMPLETED" || "$STATUS" == "PENDING" ]]; then
-    echo "The file already exists in S3. Skipping the upload."
-    continue
-  fi
+  FILE_EXISTS=0
+  existing_files=$(aws s3api list-objects --bucket "$S3_BUCKET_NAME" --prefix "$S3_FOLDER_PATH" --query 'Contents[].Key' --output text)
+  for file in $existing_files; do
+      if [[ $file == *"$time_window_of_logs"* ]]; then
+      echo "Skipping the time window: ${time_window_of_logs} as it is already covered by the ${file} within S3."
+      FILE_EXISTS=1
+      fi
+  done
 
-  # Query Loki and create part file. The part file will be created in the current directory.
-  logcli query '{job=~".+"}' --output jsonl --timezone=UTC --tls-skip-verify --from "$start_time" --to "$end_time" --parallel-max-workers=2 --parallel-duration=120m --part-path-prefix=$(pwd)/$prefix_file_name
 
-  # Check for multiple part files. This should never since each parallel-duration is 2 hours which exceeds the requested time range of 1 hour.
-  part_files_count=$(ls -1 *.part 2>/dev/null | wc -l)
+  if [ $FILE_EXISTS -eq 0 ]; then
+      echo "The ${file} does not exist in S3. Starting to fetch logs from loki now..."
 
-  if [ "$part_files_count" -gt 1 ]; then
-    echo "Error: Found multiple part files. There should only be 1 part file. Skipping to the next hour."
-    ERRORS += 1
-    cleanup
-    continue
-  fi
+      # Query Loki and create part file. The part file will be created in the current directory.
+      logcli query '{job=~".+"}' --output jsonl --timezone=UTC --tls-skip-verify --from "$start_time" --to "$end_time" --parallel-max-workers=2 --parallel-duration=120m --part-path-prefix=$(pwd)/$prefix_file_name
+
+      # Check for multiple part files. This should never since each parallel-duration is 2 hours which exceeds the requested time range of 1 hour.
+      part_files_count=$(ls -1 *.part 2>/dev/null | wc -l)
 
-  part_file=$(ls *.part | head -n 1)
+      if [ $part_files_count -gt 1 ]; then
+        echo "Error: Found multiple part files. There should only be 1 part file. Skipping to the next hour."
+        ERRORS += 1
+        cleanup
+        continue
+      fi
 
-  # Gzip and upload the part file to S3
-  gzip "$part_file"
-  aws s3 cp "${part_file}.gz" "s3://${S3_BUCKET_NAME}/${s3_path}"
+      part_file=$(ls *.part | head -n 1)
 
+      # Gzip and upload the part file to S3
+      gzip "$part_file"
+      aws s3 cp "${part_file}.gz" "s3://${S3_BUCKET_NAME}/${s3_path}"
+      echo "The ${file} has been uploaded to S3."
+
+  fi
   cleanup
 
 done