version update bug fix

DSL/CronManager/DSL/dataset_processing.yml

@@ -2,10 +2,10 @@ dataset_processor:
   trigger: off
   type: exec
   command: "../app/scripts/data_processor_exec.sh"
-  allowedEnvs: ["cookie","dgId","updateType","savedFilePath","patchPayload"]
+  allowedEnvs: ["cookie","dgId", "newDgId","updateType","savedFilePath","patchPayload"]
 
 data_validation:
   trigger: off
   type: exec
   command: "../app/scripts/data_validator_exec.sh"
-  allowedEnvs: ["cookie","dgId","updateType","savedFilePath","patchPayload"]
+  allowedEnvs: ["cookie","dgId", "newDgId","updateType","savedFilePath","patchPayload"]

DSL/CronManager/script/data_processor_exec.sh

@@ -1,20 +1,21 @@
 #!/bin/bash
 
 # Ensure required environment variables are set
-if [ -z "$dgId" ] || [ -z "$cookie" ] || [ -z "$updateType" ] || [ -z "$savedFilePath" ] || [ -z "$patchPayload" ]; then
+if [ -z "$dgId" ] || [ -z "$newDgId" ] || [ -z "$cookie" ] || [ -z "$updateType" ] || [ -z "$savedFilePath" ] || [ -z "$patchPayload" ]; then
   echo "One or more environment variables are missing."
-  echo "Please set dgId, cookie, updateType, savedFilePath, and patchPayload."
+  echo "Please set dgId, newDgId, cookie, updateType, savedFilePath, and patchPayload."
   exit 1
 fi
 
-# Construct the payload using grep
+# Construct the payload using here document
 payload=$(cat <<EOF
 {
-  "dgID": "$dgId",
-  "cookie": "$cookie",
+  "dgId": "$dgId",
+  "newDgId": "$newDgId",
   "updateType": "$updateType",
   "savedFilePath": "$savedFilePath",
-  "patchPayload": "$patchPayload"
+  "patchPayload": "$patchPayload",
+  "cookie": "$cookie"
 }
 EOF
 )

DSL/CronManager/script/data_validator_exec.sh

@@ -1,16 +1,17 @@
 #!/bin/bash
 
 # Ensure required environment variables are set
-if [ -z "$dgId" ] || [ -z "$cookie" ] || [ -z "$updateType" ] || [ -z "$savedFilePath" ] || [ -z "$patchPayload" ]; then
+if [ -z "$dgId" ] || [ -z "$newDgId" ] || [ -z "$cookie" ] || [ -z "$updateType" ] || [ -z "$savedFilePath" ] || [ -z "$patchPayload" ]; then
   echo "One or more environment variables are missing."
-  echo "Please set dgId, cookie, updateType, savedFilePath, and patchPayload."
+  echo "Please set dgId, newDgId, cookie, updateType, savedFilePath, and patchPayload."
   exit 1
 fi
 
 # Construct the payload using here document
 payload=$(cat <<EOF
 {
   "dgId": "$dgId",
+  "newDgId": "$newDgId",
   "updateType": "$updateType",
   "savedFilePath": "$savedFilePath",
   "patchPayload": "$patchPayload",

dataset-processor/dataset_processor_api.py

@@ -19,7 +19,8 @@
 )
 
 class ProcessHandlerRequest(BaseModel):
-    dgID: int
+    dgId: int
+    newDgId: int
     cookie: str
     updateType: str
     savedFilePath: str
@@ -47,7 +48,7 @@ async def process_handler_endpoint(request: Request):
     await authenticate_user(request)
 
     authCookie = payload["cookie"]
-    result = processor.process_handler(int(payload["dgID"]), authCookie, payload["updateType"], payload["savedFilePath"], payload["patchPayload"])
+    result = processor.process_handler(int(payload["dgId"]), int(payload["newDgId"]), authCookie, payload["updateType"], payload["savedFilePath"], payload["patchPayload"])
     if result:
         return result
     else:
@@ -69,6 +70,7 @@ async def forward_request(request: Request, response: Response):
     print(payload)
     payload2 = {}
     payload2["dgId"] = int(payload["dgId"])
+    payload2["newDgId"] = int(payload["newDgId"])
     payload2["updateType"] = payload["updateType"]
     payload2["patchPayload"] = payload["patchPayload"]
     payload2["savedFilePath"] = payload["savedFilePath"]

docker-compose.yml

@@ -226,7 +226,8 @@ services:
       - FILE_HANDLER_DOWNLOAD_LOCATION_JSON_URL=http://file-handler:8000/datasetgroup/data/download/json/location
       - FILE_HANDLER_STOPWORDS_URL=http://file-handler:8000/datasetgroup/data/download/json/stopwords
       - FILE_HANDLER_IMPORT_CHUNKS_URL=http://file-handler:8000/datasetgroup/data/import/chunk
-      - GET_PAGE_COUNT_URL=http://ruuter-private:8088/classifier/datasetgroup/group/data?groupId=dgID&pageNum=1
+      - FILE_HANDLER_COPY_CHUNKS_URL=http://file-handler:8000/datasetgroup/data/copy
+      - GET_PAGE_COUNT_URL=http://ruuter-private:8088/classifier/datasetgroup/group/data?groupId=dgId&pageNum=1
       - SAVE_JSON_AGGREGRATED_DATA_URL=http://file-handler:8000/datasetgroup/data/import/json
       - DOWNLOAD_CHUNK_URL=http://file-handler:8000/datasetgroup/data/download/chunk
       - STATUS_UPDATE_URL=http://ruuter-private:8088/classifier/datasetgroup/update/preprocess/status

file-handler/file_handler_api.py

@@ -42,6 +42,11 @@ class ImportJsonMajor(BaseModel):
     dgId: int
     dataset: list
 
+class CopyPayload(BaseModel):
+    dgId: int
+    newDgId: int
+    fileLocations: list
+
 if not os.path.exists(UPLOAD_DIRECTORY):
     os.makedirs(UPLOAD_DIRECTORY)
 
@@ -282,3 +287,36 @@ async def upload_and_copy(request: Request, importData: ImportJsonMajor):
         return JSONResponse(status_code=200, content=upload_success)
     else:
         raise HTTPException(status_code=500, detail=S3_UPLOAD_FAILED)
+
+@app.post("/datasetgroup/data/copy")
+async def upload_and_copy(request: Request, copyPayload: CopyPayload):
+    cookie = request.cookies.get("customJwtCookie")
+    await authenticate_user(f'customJwtCookie={cookie}')
+
+    dg_id = copyPayload.dgId
+    new_dg_id = copyPayload.newDgId
+    files = copyPayload.fileLocations
+
+    if len(files)>0:
+        local_storage_location = "temp_copy.json"
+    else:
+        print("Abort copying since sent file list does not have any entry.")
+        upload_success = UPLOAD_SUCCESS.copy()
+        upload_success["saved_file_path"] = ""
+        return JSONResponse(status_code=200, content=upload_success)
+    for file in files:
+        old_location = f"/dataset/{dg_id}/{file}"
+        new_location = f"/dataset/{new_dg_id}/{file}"
+        response = s3_ferry.transfer_file(local_storage_location, "FS", old_location, "S3")
+        response = s3_ferry.transfer_file(new_location, "S3", local_storage_location, "FS")
+
+        if response.status_code == 201:
+            print(f"Copying completed : {file}")
+        else:
+            print(f"Copying failed : {file}")
+            raise HTTPException(status_code=500, detail=S3_UPLOAD_FAILED)
+    else:
+        os.remove(local_storage_location)
+        upload_success = UPLOAD_SUCCESS.copy()
+        upload_success["saved_file_path"] = f"/dataset/{new_dg_id}/"
+        return JSONResponse(status_code=200, content=upload_success)