Merge branch 'develop' into refactor.sources-to-v2-batch2

rudderlabs · Feb 18, 2025 · bc7b3d9 · bc7b3d9
2 parents f136015 + c272267
commit bc7b3d9
Show file tree

Hide file tree

Showing 20 changed files with 1,043 additions and 335 deletions.
diff --git a/.github/workflows/build-pr-artifacts.yml b/.github/workflows/build-pr-artifacts.yml
@@ -53,7 +53,7 @@ jobs:
       load_target: development
       push_target: production
     secrets:
-      DOCKERHUB_PROD_TOKEN: ${{ secrets.DOCKERHUB_PROD_TOKEN }}
+      DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
 
   build-user-transformer-image:
     name: Build User Transformer Docker Image - PR
@@ -69,4 +69,4 @@ jobs:
       load_target: development
       push_target: production
     secrets:
-      DOCKERHUB_PROD_TOKEN: ${{ secrets.DOCKERHUB_PROD_TOKEN }}
+      DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
diff --git a/.github/workflows/build-push-docker-image.yml b/.github/workflows/build-push-docker-image.yml
@@ -33,7 +33,7 @@ on:
       workflow_url:
         type: string
     secrets:
-      DOCKERHUB_PROD_TOKEN:
+      DOCKERHUB_TOKEN:
         required: true
 
 env:
@@ -101,7 +101,7 @@ jobs:
         uses: docker/login-action@v3.3.0
         with:
           username: ${{ env.DOCKERHUB_USERNAME }}
-          password: ${{ secrets.DOCKERHUB_PROD_TOKEN }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
 
       - name: Setup Docker Buildx
         uses: docker/setup-buildx-action@v3.7.1
@@ -154,7 +154,7 @@ jobs:
         uses: docker/login-action@v3.3.0
         with:
           username: ${{ env.DOCKERHUB_USERNAME }}
-          password: ${{ secrets.DOCKERHUB_PROD_TOKEN }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
 
       - name: Setup Docker Buildx
         uses: docker/setup-buildx-action@v3.7.1
@@ -202,7 +202,7 @@ jobs:
         uses: docker/login-action@v3.3.0
         with:
           username: ${{ env.DOCKERHUB_USERNAME }}
-          password: ${{ secrets.DOCKERHUB_PROD_TOKEN }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
 
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v3.7.1

diff --git a/.github/workflows/prepare-for-dev-deploy.yml b/.github/workflows/prepare-for-dev-deploy.yml
@@ -62,7 +62,7 @@ jobs:
       push_target: production
       use_merge_sha: true
     secrets:
-      DOCKERHUB_PROD_TOKEN: ${{ secrets.DOCKERHUB_PROD_TOKEN }}
+      DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
 
   build-user-transformer-image:
     name: Build User Transformer Docker Image - Dev
@@ -78,4 +78,4 @@ jobs:
       load_target: development
       push_target: production
     secrets:
-      DOCKERHUB_PROD_TOKEN: ${{ secrets.DOCKERHUB_PROD_TOKEN }}
+      DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
diff --git a/.github/workflows/prepare-for-prod-dt-deploy.yml b/.github/workflows/prepare-for-prod-dt-deploy.yml
@@ -60,7 +60,7 @@ jobs:
       use_merge_sha: true
       skip_tests: ${{startsWith(github.event.pull_request.head.ref, 'hotfix-release/')}}
     secrets:
-      DOCKERHUB_PROD_TOKEN: ${{ secrets.DOCKERHUB_PROD_TOKEN }}
+      DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
 
   create-pull-request:
     name: Update Helm Charts For Production and Create Pull Request

diff --git a/.github/workflows/prepare-for-prod-ut-deploy.yml b/.github/workflows/prepare-for-prod-ut-deploy.yml
@@ -64,7 +64,7 @@ jobs:
       use_merge_sha: true
       skip_tests: ${{startsWith(github.event.pull_request.head.ref, 'hotfix-release/')}}
     secrets:
-      DOCKERHUB_PROD_TOKEN: ${{ secrets.DOCKERHUB_PROD_TOKEN }}
+      DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
 
   create-pull-request:
     name: Update Helm Charts For Production and Create Pull Request

diff --git a/.github/workflows/prepare-for-staging-deploy.yml b/.github/workflows/prepare-for-staging-deploy.yml
@@ -54,7 +54,7 @@ jobs:
       push_target: production
       use_merge_sha: true
     secrets:
-      DOCKERHUB_PROD_TOKEN: ${{ secrets.DOCKERHUB_PROD_TOKEN }}
+      DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
 
   build-user-transformer-image:
     name: Build User Transformer Docker Image - Staging
@@ -71,7 +71,7 @@ jobs:
       load_target: development
       push_target: production
     secrets:
-      DOCKERHUB_PROD_TOKEN: ${{ secrets.DOCKERHUB_PROD_TOKEN }}
+      DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
 
   create-pull-request:
     name: Update Helm Charts For Staging and Create Pull Request

diff --git a/package-lock.json b/package-lock.json
diff --git a/package.json b/package.json
@@ -92,7 +92,7 @@
     "json-diff": "^1.0.3",
     "json-size": "^1.0.0",
     "jsontoxml": "^1.0.1",
-    "koa": "^2.15.3",
+    "koa": "^2.15.4",
     "koa-bodyparser": "^4.4.0",
     "koa2-swagger-ui": "^5.7.0",
     "libphonenumber-js": "^1.11.18",

diff --git a/src/cdk/v2/destinations/http/procWorkflow.yaml b/src/cdk/v2/destinations/http/procWorkflow.yaml
@@ -55,7 +55,8 @@ steps:
 
   - name: prepareBody
     template: |
-      const payload = $.getCustomMappings(.message, .destination.Config.propertiesMapping);
+      const propertiesMapping = .destination.Config.isDefaultMapping ? [{"to": "$", "from": "$"}] : .destination.Config.propertiesMapping;
+      const payload = $.getCustomMappings(.message, propertiesMapping);
       $.context.payload = $.prepareBody(payload, $.context.format, .destination.Config.xmlRootKey);
 
   - name: buildResponseForProcessTransformation

diff --git a/src/cdk/v2/destinations/zoho/config.js b/src/cdk/v2/destinations/zoho/config.js
@@ -10,8 +10,8 @@ const DATA_CENTRE_BASE_ENDPOINTS_MAP = {
 };
 
 const getBaseEndpoint = (dataServer) => DATA_CENTRE_BASE_ENDPOINTS_MAP[dataServer];
-const COMMON_RECORD_ENDPOINT = (dataCenter = 'US') =>
-  `${getBaseEndpoint(dataCenter)}/crm/v6/moduleType`;
+const COMMON_RECORD_ENDPOINT = (dataCenter) =>
+  `${getBaseEndpoint(dataCenter || 'US')}/crm/v6/moduleType`;
 
 // ref: https://www.zoho.com/crm/developer/docs/api/v6/insert-records.html#:~:text=%2DX%20POST-,System%2Ddefined%20mandatory%20fields%20for%20each%20module,-While%20inserting%20records
 const MODULE_MANDATORY_FIELD_CONFIG = {

diff --git a/src/cdk/v2/destinations/zoho/transformRecord.js b/src/cdk/v2/destinations/zoho/transformRecord.js
@@ -33,7 +33,7 @@ const responseBuilder = (
   identifierType,
   operationModuleType,
   commonEndPoint,
-  action,
+  isUpsert,
   metadata,
 ) => {
   const { trigger, addDefaultDuplicateCheck, multiSelectFieldLevelDecision } = config;
@@ -43,7 +43,7 @@ const responseBuilder = (
     Authorization: `Zoho-oauthtoken ${metadata[0].secret.accessToken}`,
   };
 
-  if (action === 'insert' || action === 'update') {
+  if (isUpsert) {
     const payload = {
       duplicate_check_fields: handleDuplicateCheck(
         addDefaultDuplicateCheck,
@@ -70,7 +70,6 @@ const batchResponseBuilder = (
   identifierType,
   operationModuleType,
   upsertEndPoint,
-  action,
 ) => {
   const upsertResponseArray = [];
   const deletionResponseArray = [];
@@ -101,7 +100,7 @@ const batchResponseBuilder = (
         identifierType,
         operationModuleType,
         upsertEndPoint,
-        action,
+        true,
         upsertmetadataChunks.items[0],
       ),
     );
@@ -115,7 +114,7 @@ const batchResponseBuilder = (
         identifierType,
         operationModuleType,
         upsertEndPoint,
-        action,
+        false,
         deletionmetadataChunks.items[0],
       ),
     );
@@ -226,13 +225,12 @@ const handleDeletion = async (
  */
 const processInput = async (
   input,
-  action,
   operationModuleType,
   Config,
   transformedResponseToBeBatched,
   errorResponseList,
 ) => {
-  const { fields } = input.message;
+  const { fields, action } = input.message;
 
   if (isEmptyObject(fields)) {
     const emptyFieldsError = new InstrumentationError('`fields` cannot be empty');
@@ -285,7 +283,6 @@ const processRecordInputs = async (inputs, destination) => {
   const response = [];
   const errorResponseList = [];
   const { Config } = destination;
-  const { action } = inputs[0].message;
 
   const transformedResponseToBeBatched = {
     upsertData: [],
@@ -296,13 +293,12 @@ const processRecordInputs = async (inputs, destination) => {
 
   const { operationModuleType, identifierType, upsertEndPoint } = deduceModuleInfo(inputs, Config);
 
-  validateConfigurationIssue(Config, operationModuleType, action);
+  validateConfigurationIssue(Config, operationModuleType);
 
   await Promise.all(
     inputs.map((input) =>
       processInput(
         input,
-        action,
         operationModuleType,
         Config,
         transformedResponseToBeBatched,
@@ -322,7 +318,6 @@ const processRecordInputs = async (inputs, destination) => {
     identifierType,
     operationModuleType,
     upsertEndPoint,
-    action,
   );
 
   if (upsertResponseArray.length === 0 && deletionResponseArray.length === 0) {