From fd585ab587e3af81bbf331c8290f61df2685759b Mon Sep 17 00:00:00 2001
From: xyfer17 <nav3044999@gmail.com>
Date: Sat, 26 Oct 2024 22:34:19 +0530
Subject: [PATCH 01/48] fix(CE): form-data loss on navigation

---
 ui/src/components/ExitModal/ExitModal.tsx     |  9 ++-
 .../JSONSchemaForm/JSONSchemaForm.tsx         | 37 ++++++++++-
 ui/src/stores/useConnectorFormStore.ts        | 61 +++++++++++++++++++
 .../ConnectorConfigForm.tsx                   |  2 +
 .../EditDestinations/EditDestinations.tsx     |  1 +
 .../Sources/EditSource/EditSource.tsx         |  1 +
 6 files changed, 107 insertions(+), 4 deletions(-)
 create mode 100644 ui/src/stores/useConnectorFormStore.ts

diff --git a/ui/src/components/ExitModal/ExitModal.tsx b/ui/src/components/ExitModal/ExitModal.tsx
index 1b756eee..31ae93d1 100644
--- a/ui/src/components/ExitModal/ExitModal.tsx
+++ b/ui/src/components/ExitModal/ExitModal.tsx
@@ -14,11 +14,18 @@ import {
 } from '@chakra-ui/react';
 import { useNavigate } from 'react-router-dom';
 import ExitWarningImage from '@/assets/images/ExitWarning.png';
+import useConnectorFormStore from '@/stores/useConnectorFormStore';
 
 const ExitModal = (): JSX.Element => {
   const { isOpen, onOpen, onClose } = useDisclosure();
+  const { resetAllFormData } = useConnectorFormStore();
   const navigate = useNavigate();
 
+  const handleSetupExit = () => {
+    resetAllFormData();
+    navigate('*');
+  };
+
   return (
     <>
       <Button
@@ -74,7 +81,7 @@ const ExitModal = (): JSX.Element => {
                   variant='solid'
                   color='white'
                   rounded='lg'
-                  onClick={() => navigate('*')}
+                  onClick={handleSetupExit}
                   letterSpacing='-0.14px'
                 >
                   Exit
diff --git a/ui/src/components/JSONSchemaForm/JSONSchemaForm.tsx b/ui/src/components/JSONSchemaForm/JSONSchemaForm.tsx
index dda38f87..1cee1032 100644
--- a/ui/src/components/JSONSchemaForm/JSONSchemaForm.tsx
+++ b/ui/src/components/JSONSchemaForm/JSONSchemaForm.tsx
@@ -1,3 +1,4 @@
+import { useEffect } from 'react';
 import Form from '@rjsf/chakra-ui';
 import { RJSFSchema } from '@rjsf/utils';
 import validator from '@rjsf/validator-ajv8';
@@ -9,6 +10,7 @@ import BaseInputTemplate from '@/components/JSONSchemaForm/rjsf/BaseInputTemplat
 import DescriptionFieldTemplate from '@/components/JSONSchemaForm/rjsf/DescriptionFieldTemplate';
 import { FormProps } from '@rjsf/core';
 import WrapIfAdditionalTemplate from './rjsf/WrapIfAdditionalTemplate';
+import useConnectorFormStore from '@/stores/useConnectorFormStore';
 
 type JSONSchemaFormProps = {
   schema: RJSFSchema;
@@ -17,6 +19,8 @@ type JSONSchemaFormProps = {
   onChange?: (formData: FormData) => void;
   children?: JSX.Element;
   formData?: unknown;
+  connectorId?: string;
+  connectorType: string;
 };
 
 const JSONSchemaForm = ({
@@ -26,7 +30,13 @@ const JSONSchemaForm = ({
   onChange,
   children,
   formData,
+  connectorId,
+  connectorType,
 }: JSONSchemaFormProps): JSX.Element => {
+  const { getConnectorFormData, setConnectorFormData, resetConnectorFormData } =
+    useConnectorFormStore();
+  const currentFormData = connectorId ? getConnectorFormData(connectorType, connectorId) : formData;
+
   const templateOverrides: FormProps<any, RJSFSchema, any>['templates'] = {
     ObjectFieldTemplate: ObjectFieldTemplate,
     TitleFieldTemplate: TitleFieldTemplate,
@@ -35,15 +45,36 @@ const JSONSchemaForm = ({
     DescriptionFieldTemplate: DescriptionFieldTemplate,
     WrapIfAdditionalTemplate: WrapIfAdditionalTemplate,
   };
+
+  const handleFormChange = (data: any) => {
+    const updatedFormData = data.formData;
+    connectorId && setConnectorFormData(connectorType, connectorId, updatedFormData);
+    onChange?.(updatedFormData);
+  };
+
+  useEffect(() => {
+    const handleBeforeUnload = () => {
+      connectorId && resetConnectorFormData(connectorType, connectorId);
+    };
+    window.addEventListener('beforeunload', handleBeforeUnload);
+
+    return () => {
+      window.removeEventListener('beforeunload', handleBeforeUnload);
+    };
+  }, [connectorType, connectorId, resetConnectorFormData]);
+
   return (
     <Form
       uiSchema={uiSchema}
       schema={schema}
       validator={validator}
       templates={templateOverrides}
-      formData={formData}
-      onSubmit={({ formData }) => onSubmit(formData)}
-      onChange={({ formData }) => onChange?.(formData)}
+      formData={currentFormData}
+      onSubmit={({ formData }) => {
+        onSubmit(formData);
+        connectorId && resetConnectorFormData(connectorType, connectorId);
+      }}
+      onChange={handleFormChange}
     >
       {children}
     </Form>
diff --git a/ui/src/stores/useConnectorFormStore.ts b/ui/src/stores/useConnectorFormStore.ts
new file mode 100644
index 00000000..688cfe1c
--- /dev/null
+++ b/ui/src/stores/useConnectorFormStore.ts
@@ -0,0 +1,61 @@
+import { create } from 'zustand';
+
+type ConnectorType = string;
+type ConnectorId = string;
+interface FormData {
+  [key: string]: unknown;
+}
+
+interface ConnectorFormState {
+  formsData: {
+    [connectorType: string]: {
+      [connectorId: string]: FormData;
+    };
+  };
+  setConnectorFormData: (
+    connectorType: ConnectorType,
+    connectorId: ConnectorId,
+    data: FormData,
+  ) => void;
+  getConnectorFormData: (connectorType: ConnectorType, connectorId: ConnectorId) => FormData | null;
+  resetConnectorFormData: (connectorType: ConnectorType, connectorId: ConnectorId) => void;
+  resetAllFormData: () => void;
+}
+
+const useConnectorFormStore = create<ConnectorFormState>((set, get) => ({
+  formsData: {},
+
+  setConnectorFormData: (connectorType, connectorId, data) =>
+    set((state) => ({
+      formsData: {
+        ...state.formsData,
+        [connectorType]: {
+          ...state.formsData[connectorType],
+          [connectorId]: data,
+        },
+      },
+    })),
+
+  getConnectorFormData: (connectorType, connectorId) => {
+    const state = get();
+    return state.formsData[connectorType]?.[connectorId] ?? null;
+  },
+
+  resetConnectorFormData: (connectorType, connectorId) =>
+    set((state) => {
+      if (!state.formsData[connectorType]) return state;
+      const remainingConnectorData = Object.fromEntries(
+        Object.entries(state.formsData[connectorType] || {}).filter(([key]) => key !== connectorId),
+      );
+      return {
+        formsData: {
+          ...state.formsData,
+          [connectorType]: remainingConnectorData,
+        },
+      };
+    }),
+
+  resetAllFormData: () => set(() => ({ formsData: {} })),
+}));
+
+export default useConnectorFormStore;
diff --git a/ui/src/views/Connectors/ConnectorConfigForm/ConnectorConfigForm.tsx b/ui/src/views/Connectors/ConnectorConfigForm/ConnectorConfigForm.tsx
index c1996394..a24cf7d8 100644
--- a/ui/src/views/Connectors/ConnectorConfigForm/ConnectorConfigForm.tsx
+++ b/ui/src/views/Connectors/ConnectorConfigForm/ConnectorConfigForm.tsx
@@ -55,6 +55,8 @@ const ConnectorConfigForm = ({ connectorType }: { connectorType: string }): JSX.
             schema={connectorSchema}
             uiSchema={generatedSchema}
             onSubmit={(formData: FormData) => handleFormSubmit(formData)}
+            connectorId={connector}
+            connectorType={connectorType}
           >
             <FormFooter
               ctaName='Continue'
diff --git a/ui/src/views/Connectors/Destinations/EditDestinations/EditDestinations.tsx b/ui/src/views/Connectors/Destinations/EditDestinations/EditDestinations.tsx
index 657ff9ae..1b5ab10a 100644
--- a/ui/src/views/Connectors/Destinations/EditDestinations/EditDestinations.tsx
+++ b/ui/src/views/Connectors/Destinations/EditDestinations/EditDestinations.tsx
@@ -213,6 +213,7 @@ const EditDestination = (): JSX.Element => {
             formData={formData}
             onSubmit={(formData: FormData) => handleOnTestClick(formData)}
             onChange={(formData: FormData) => setFormData(formData)}
+            connectorType='destination'
           >
             <FormFooter
               ctaName='Save Changes'
diff --git a/ui/src/views/Connectors/Sources/EditSource/EditSource.tsx b/ui/src/views/Connectors/Sources/EditSource/EditSource.tsx
index 64dae93e..e683c8fb 100644
--- a/ui/src/views/Connectors/Sources/EditSource/EditSource.tsx
+++ b/ui/src/views/Connectors/Sources/EditSource/EditSource.tsx
@@ -209,6 +209,7 @@ const EditSource = (): JSX.Element => {
             formData={formData}
             onSubmit={(formData: FormData) => handleOnTestClick(formData)}
             onChange={(formData: FormData) => setFormData(formData)}
+            connectorType='source'
           >
             <FormFooter
               ctaName='Save Changes'

From dcf1c3b7ee0dd88790b23b2af875979a3e5806dd Mon Sep 17 00:00:00 2001
From: "github-actions[bot]"
 <41898282+github-actions[bot]@users.noreply.github.com>
Date: Thu, 24 Oct 2024 11:22:00 +0530
Subject: [PATCH 02/48] chore(CE): add audit logs serializer and audit log
 permission (#426)

Co-authored-by: TivonB-AI2 <124182151+TivonB-AI2@users.noreply.github.com>
---
 .../app/serializers/audit_logs_serializer.rb  |  7 +++
 ...3042_add_audit_logs_permission_to_roles.rb | 63 +++++++++++++++++++
 server/db/data_schema.rb                      |  2 +-
 3 files changed, 71 insertions(+), 1 deletion(-)
 create mode 100644 server/app/serializers/audit_logs_serializer.rb
 create mode 100644 server/db/data/20241022143042_add_audit_logs_permission_to_roles.rb

diff --git a/server/app/serializers/audit_logs_serializer.rb b/server/app/serializers/audit_logs_serializer.rb
new file mode 100644
index 00000000..83c0f785
--- /dev/null
+++ b/server/app/serializers/audit_logs_serializer.rb
@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+# app/serializers/audit_logs_serializer.rb
+class AuditLogsSerializer < ActiveModel::Serializer
+  attributes :id, :user_id, :action, :resource_type, :resource_id, :resource, :workspace_id,
+             :metadata
+end
diff --git a/server/db/data/20241022143042_add_audit_logs_permission_to_roles.rb b/server/db/data/20241022143042_add_audit_logs_permission_to_roles.rb
new file mode 100644
index 00000000..c6a314d6
--- /dev/null
+++ b/server/db/data/20241022143042_add_audit_logs_permission_to_roles.rb
@@ -0,0 +1,63 @@
+# frozen_string_literal: true
+
+class AddAuditLogsPermissionToRoles < ActiveRecord::Migration[7.1]
+  def change # rubocop:disable Metrics/MethodLength
+    admin_role = Role.find_by(role_name: "Admin")
+    member_role = Role.find_by(role_name: "Member")
+    viewer_role = Role.find_by(role_name: "Viewer")
+
+    admin_role&.update!(
+      policies: {
+        permissions: {
+          connector_definition: { create: true, read: true, update: true, delete: true },
+          connector: { create: true, read: true, update: true, delete: true },
+          model: { create: true, read: true, update: true, delete: true },
+          report: { create: true, read: true, update: true, delete: true },
+          sync_record: { create: true, read: true, update: true, delete: true },
+          sync_run: { create: true, read: true, update: true, delete: true },
+          sync: { create: true, read: true, update: true, delete: true },
+          user: { create: true, read: true, update: true, delete: true },
+          workspace: { create: true, read: true, update: true, delete: true },
+          data_app: { create: true, read: true, update: true, delete: true },
+          audit_logs: { create: true, read: true, update: true, delete: true }
+        }
+      }
+    )
+
+    member_role&.update!(
+      policies: {
+        permissions: {
+          connector_definition: { create: true, read: true, update: true, delete: true },
+          connector: { create: true, read: true, update: true, delete: true },
+          model: { create: true, read: true, update: true, delete: true },
+          report: { create: true, read: true, update: true, delete: true },
+          sync_record: { create: true, read: true, update: true, delete: true },
+          sync_run: { create: true, read: true, update: true, delete: true },
+          sync: { create: true, read: true, update: true, delete: true },
+          user: { create: false, read: true, update: false, delete: false },
+          workspace: { create: false, read: true, update: false, delete: false },
+          data_app: { create: true, read: true, update: true, delete: true },
+          audit_logs: { create: true, read: true, update: true, delete: true }
+        }
+      }
+    )
+
+    viewer_role&.update!(
+      policies: {
+        permissions: {
+          connector_definition: { create: false, read: true, update: false, delete: false },
+          connector: { create: false, read: true, update: false, delete: false },
+          model: { create: false, read: true, update: false, delete: false },
+          report: { create: false, read: true, update: false, delete: false },
+          sync_record: { create: false, read: true, update: false, delete: false },
+          sync_run: { create: false, read: true, update: false, delete: false },
+          sync: { create: false, read: true, update: false, delete: false },
+          user: { create: false, read: true, update: false, delete: false },
+          workspace: { create: false, read: true, update: false, delete: false },
+          data_app: { create: false, read: true, update: false, delete: false },
+          audit_logs: { create: false, read: false, update: false, delete: false }
+        }
+      }
+    )
+  end
+end
diff --git a/server/db/data_schema.rb b/server/db/data_schema.rb
index ba8761bd..5c703a1d 100644
--- a/server/db/data_schema.rb
+++ b/server/db/data_schema.rb
@@ -1,3 +1,3 @@
 # frozen_string_literal: true
 
-DataMigrate::Data.define(version: 20_240_903_055_705)
+DataMigrate::Data.define(version: 20_241_022_143_042)

From ef1c394a3c26f0d4919fbbe67a35b40bc0865edb Mon Sep 17 00:00:00 2001
From: datafloyd <subinthattaparambil@gmail.com>
Date: Thu, 24 Oct 2024 16:56:43 +0400
Subject: [PATCH 03/48] fix(CE): add exception to tracker (#436)

---
 server/lib/script_vault/tracker.rb | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/server/lib/script_vault/tracker.rb b/server/lib/script_vault/tracker.rb
index b5352263..d1183f51 100644
--- a/server/lib/script_vault/tracker.rb
+++ b/server/lib/script_vault/tracker.rb
@@ -12,7 +12,7 @@ def self.included(base)
           request.body = event_data.to_json
           http.request(request)
         end
-      rescue StandardError
+      rescue StandardError => e
         Rails.logger.error "Failed to transmit data: #{e.message}"
       end
       base.send(:private, :dispatch_details)
@@ -34,7 +34,7 @@ def _track_event(event_name, properties = {})
 
       external_event_data = event_data.dup
       dispatch_details(external_event_data)
-    rescue StandardError
+    rescue StandardError => e
       Rails.logger.error "Failed to transmit data: #{e.message}"
     end
   end

From 9f7f06ca5566f3da7188fb95dd60e409bdcd2a9e Mon Sep 17 00:00:00 2001
From: "github-actions[bot]"
 <41898282+github-actions[bot]@users.noreply.github.com>
Date: Fri, 25 Oct 2024 10:38:51 +0530
Subject: [PATCH 04/48] chore(CE): added audit logs to catalogs_controller
 (#439)

Co-authored-by: TivonB-AI2 <124182151+TivonB-AI2@users.noreply.github.com>
---
 .../controllers/api/v1/catalogs_controller.rb | 11 ++++++
 .../api/v1/catalogs_controller_spec.rb        | 36 +++++++++++++++++++
 2 files changed, 47 insertions(+)

diff --git a/server/app/controllers/api/v1/catalogs_controller.rb b/server/app/controllers/api/v1/catalogs_controller.rb
index c93d60e5..37818367 100644
--- a/server/app/controllers/api/v1/catalogs_controller.rb
+++ b/server/app/controllers/api/v1/catalogs_controller.rb
@@ -4,9 +4,12 @@ module Api
   module V1
     class CatalogsController < ApplicationController
       include Catalogs
+      include AuditLogger
       before_action :set_connector, only: %i[create update]
       before_action :set_catalog, only: %i[update]
 
+      after_action :create_audit_log
+
       def create
         authorize current_workspace, policy_class: ConnectorPolicy
         result = CreateCatalog.call(
@@ -16,6 +19,8 @@ def create
 
         if result.success?
           @catalog = result.catalog
+          @audit_resource = @catalog.catalog["streams"].first["name"]
+          @payload = catalog_params.to_h
           render json: @catalog, status: :created
         else
           render_error(
@@ -36,6 +41,8 @@ def update
 
         if result.success?
           @catalog = result.catalog
+          @audit_resource = @catalog.catalog["streams"].first["name"]
+          @payload = catalog_params.to_h
           render json: @catalog, status: :created
         else
           render_error(
@@ -61,6 +68,10 @@ def set_catalog
         @catalog = @connector.catalog
       end
 
+      def create_audit_log
+        audit!(resource_id: params[:id], resource: @audit_resource, payload: @payload)
+      end
+
       def catalog_params
         params.require(:catalog).permit(json_schema: {})
       end
diff --git a/server/spec/requests/api/v1/catalogs_controller_spec.rb b/server/spec/requests/api/v1/catalogs_controller_spec.rb
index 52b684ca..066e2728 100644
--- a/server/spec/requests/api/v1/catalogs_controller_spec.rb
+++ b/server/spec/requests/api/v1/catalogs_controller_spec.rb
@@ -87,6 +87,15 @@
         expect(response_hash.dig(:data, :attributes, :catalog, :streams).first["name"]).to eq(connector.name)
         expect(response_hash.dig(:data, :attributes, :catalog,
                                  :streams).first["json_schema"]).to eq(request_body[:catalog]["json_schema"])
+
+        audit_log = AuditLog.last
+        expect(audit_log).not_to be_nil
+        expect(audit_log.user_id).to eq(user.id)
+        expect(audit_log.action).to eq("create")
+        expect(audit_log.resource_type).to eq("Catalog")
+        expect(audit_log.resource_id).to eq(nil)
+        expect(audit_log.resource).to eq(connector.name)
+        expect(audit_log.workspace_id).to eq(workspace.id)
       end
     end
 
@@ -105,6 +114,15 @@
         expect(response_hash.dig(:data, :attributes, :catalog, :streams).first["name"]).to eq(connector.name)
         expect(response_hash.dig(:data, :attributes, :catalog,
                                  :streams).first["json_schema"]).to eq(request_body[:catalog]["json_schema"])
+
+        audit_log = AuditLog.last
+        expect(audit_log).not_to be_nil
+        expect(audit_log.user_id).to eq(user.id)
+        expect(audit_log.action).to eq("create")
+        expect(audit_log.resource_type).to eq("Catalog")
+        expect(audit_log.resource_id).to eq(nil)
+        expect(audit_log.resource).to eq(connector.name)
+        expect(audit_log.workspace_id).to eq(workspace.id)
       end
     end
 
@@ -145,6 +163,15 @@
         expect(response_hash.dig(:data, :attributes, :connector_id)).to eq(connector.id)
         expect(response_hash.dig(:data, :attributes, :catalog,
                                  :streams).first["json_schema"]).to eq(update_request_body[:catalog]["json_schema"])
+
+        audit_log = AuditLog.last
+        expect(audit_log).not_to be_nil
+        expect(audit_log.user_id).to eq(user.id)
+        expect(audit_log.action).to eq("update")
+        expect(audit_log.resource_type).to eq("Catalog")
+        expect(audit_log.resource_id).to eq(existing_catalog.id)
+        expect(audit_log.resource).to eq(connector.name)
+        expect(audit_log.workspace_id).to eq(workspace.id)
       end
     end
 
@@ -164,6 +191,15 @@
         expect(response_hash.dig(:data, :attributes, :connector_id)).to eq(connector.id)
         expect(response_hash.dig(:data, :attributes, :catalog,
                                  :streams).first["json_schema"]).to eq(update_request_body[:catalog]["json_schema"])
+
+        audit_log = AuditLog.last
+        expect(audit_log).not_to be_nil
+        expect(audit_log.user_id).to eq(user.id)
+        expect(audit_log.action).to eq("update")
+        expect(audit_log.resource_type).to eq("Catalog")
+        expect(audit_log.resource_id).to eq(existing_catalog.id)
+        expect(audit_log.resource).to eq(connector.name)
+        expect(audit_log.workspace_id).to eq(workspace.id)
       end
     end
 

From e7fd2c774498289c5946f2da7bb39985784e49a0 Mon Sep 17 00:00:00 2001
From: "github-actions[bot]"
 <41898282+github-actions[bot]@users.noreply.github.com>
Date: Fri, 25 Oct 2024 10:39:32 +0530
Subject: [PATCH 05/48] chore(CE): added audit logs to schedule_sync_controller
 (#440)

Co-authored-by: TivonB-AI2 <124182151+TivonB-AI2@users.noreply.github.com>
---
 .../api/v1/schedule_syncs_controller.rb        |  9 +++++++++
 .../api/v1/schedule_syncs_controller_spec.rb   | 18 ++++++++++++++++++
 2 files changed, 27 insertions(+)

diff --git a/server/app/controllers/api/v1/schedule_syncs_controller.rb b/server/app/controllers/api/v1/schedule_syncs_controller.rb
index 7944f3a2..8a5c563c 100644
--- a/server/app/controllers/api/v1/schedule_syncs_controller.rb
+++ b/server/app/controllers/api/v1/schedule_syncs_controller.rb
@@ -4,15 +4,19 @@ module Api
   module V1
     class ScheduleSyncsController < ApplicationController
       include Syncs
+      include AuditLogger
       before_action :set_sync
       before_action :validate_sync_status
       before_action :validate_sync_schedule_type
 
+      after_action :create_audit_log
+
       def create
         authorize @sync
         result = ScheduleSync.call(sync: @sync)
 
         if result.success?
+          @audit_resource = @sync.name
           render json: { message: "Sync scheduled successfully" }, status: :ok
         else
           render_error(message: result.message, status: result.status)
@@ -24,6 +28,7 @@ def destroy
         result = CancelSync.call(sync: @sync)
 
         if result.success?
+          @audit_resource = @sync.name
           render json: { message: "Sync cancelled successfully" }, status: :ok
         else
           render_error(message: result.message, status: result.status)
@@ -44,6 +49,10 @@ def validate_sync_status
                      status: :failed_dependency)
       end
 
+      def create_audit_log
+        audit!(resource_id: params[:id], resource: @audit_resource, payload: @payload)
+      end
+
       def validate_sync_schedule_type
         return if @sync.schedule_type == "manual"
 
diff --git a/server/spec/requests/api/v1/schedule_syncs_controller_spec.rb b/server/spec/requests/api/v1/schedule_syncs_controller_spec.rb
index f4ef0689..2b555887 100644
--- a/server/spec/requests/api/v1/schedule_syncs_controller_spec.rb
+++ b/server/spec/requests/api/v1/schedule_syncs_controller_spec.rb
@@ -58,6 +58,15 @@
         )
         expect(response).to have_http_status(:ok)
         expect(JSON.parse(response.body)).to eq({ "message" => "Sync scheduled successfully" })
+
+        audit_log = AuditLog.last
+        expect(audit_log).not_to be_nil
+        expect(audit_log.user_id).to eq(user.id)
+        expect(audit_log.action).to eq("create")
+        expect(audit_log.resource_type).to eq("Schedule_sync")
+        expect(audit_log.resource_id).to eq(nil)
+        expect(audit_log.resource).to eq(request_body.dig(:sync, :name))
+        expect(audit_log.workspace_id).to eq(workspace.id)
       end
     end
 
@@ -138,6 +147,15 @@
         expect(response).to have_http_status(:ok)
         expect(JSON.parse(response.body)).to eq({ "message" => "Sync cancelled successfully" })
         expect(sync.sync_runs.last.status).to eq("canceled")
+
+        audit_log = AuditLog.last
+        expect(audit_log).not_to be_nil
+        expect(audit_log.user_id).to eq(user.id)
+        expect(audit_log.action).to eq("destroy")
+        expect(audit_log.resource_type).to eq("Schedule_sync")
+        expect(audit_log.resource_id).to eq(nil)
+        expect(audit_log.resource).to eq(request_body.dig(:sync, :name))
+        expect(audit_log.workspace_id).to eq(workspace.id)
       end
     end
 

From 23b9de36749bf7718e8c65f378f89e5f44245824 Mon Sep 17 00:00:00 2001
From: "github-actions[bot]"
 <41898282+github-actions[bot]@users.noreply.github.com>
Date: Fri, 25 Oct 2024 10:40:11 +0530
Subject: [PATCH 06/48] chore(CE): add data app session model (#438)

Co-authored-by: afthab vp <afthab@multiwoven.com>
---
 server/app/models/data_app.rb                 |  1 +
 server/app/models/data_app_session.rb         | 24 +++++++++
 server/app/models/workspace.rb                |  1 +
 ...20241024064358_create_data_app_sessions.rb | 14 +++++
 ...ount_to_feedbacks_and_visual_components.rb | 11 ++++
 server/db/schema.rb                           | 16 +++++-
 server/spec/factories/data_app_sessions.rb    |  9 ++++
 server/spec/models/data_app_session_spec.rb   | 54 +++++++++++++++++++
 server/spec/models/data_app_spec.rb           |  1 +
 server/spec/models/workspace_spec.rb          |  1 +
 10 files changed, 131 insertions(+), 1 deletion(-)
 create mode 100644 server/app/models/data_app_session.rb
 create mode 100644 server/db/migrate/20241024064358_create_data_app_sessions.rb
 create mode 100644 server/db/migrate/20241024103527_add_session_id_and_session_count_to_feedbacks_and_visual_components.rb
 create mode 100644 server/spec/factories/data_app_sessions.rb
 create mode 100644 server/spec/models/data_app_session_spec.rb

diff --git a/server/app/models/data_app.rb b/server/app/models/data_app.rb
index ad6550f4..b6e42637 100644
--- a/server/app/models/data_app.rb
+++ b/server/app/models/data_app.rb
@@ -12,6 +12,7 @@ class DataApp < ApplicationRecord
   belongs_to :workspace
   has_many :visual_components, dependent: :destroy
   has_many :models, through: :visual_components
+  has_many :data_app_sessions, dependent: :nullify
 
   after_initialize :set_default_status, if: :new_record?
 
diff --git a/server/app/models/data_app_session.rb b/server/app/models/data_app_session.rb
new file mode 100644
index 00000000..5cf4fa39
--- /dev/null
+++ b/server/app/models/data_app_session.rb
@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+class DataAppSession < ApplicationRecord
+  belongs_to :data_app
+  belongs_to :workspace
+  validates :session_id, presence: true, uniqueness: true
+  validates :data_app_id, :workspace_id, presence: true
+
+  before_create :set_times
+
+  scope :active, -> { where("end_time IS NULL OR end_time > ?", Time.zone.now) }
+
+  def expired?
+    end_time.present? && end_time <= Time.zone.now
+  end
+
+  private
+
+  def set_times
+    self.start_time = Time.zone.now
+    session_length_minutes = (ENV["DATA_SESSION_LENGTH_MINUTES"] || 10).to_i # Default to 10 minutes if not set
+    self.end_time = start_time + session_length_minutes.minutes
+  end
+end
diff --git a/server/app/models/workspace.rb b/server/app/models/workspace.rb
index 97410b97..7e3a21e3 100644
--- a/server/app/models/workspace.rb
+++ b/server/app/models/workspace.rb
@@ -21,6 +21,7 @@ class Workspace < ApplicationRecord
   has_many :syncs, dependent: :nullify
   has_many :sync_runs, dependent: :nullify
   has_many :data_apps, dependent: :nullify
+  has_many :data_app_sessions, dependent: :nullify
   belongs_to :organization
 
   STATUS_ACTIVE = "active"
diff --git a/server/db/migrate/20241024064358_create_data_app_sessions.rb b/server/db/migrate/20241024064358_create_data_app_sessions.rb
new file mode 100644
index 00000000..13c7cb64
--- /dev/null
+++ b/server/db/migrate/20241024064358_create_data_app_sessions.rb
@@ -0,0 +1,14 @@
+class CreateDataAppSessions < ActiveRecord::Migration[7.1]
+  def change
+    create_table :data_app_sessions do |t|
+      t.string :session_id, null: false
+      t.bigint :data_app_id, null: false
+      t.integer :workspace_id, null: false
+      t.datetime :start_time, null: false
+      t.datetime :end_time
+      t.timestamps
+    end
+    add_index :data_app_sessions, :session_id, unique: true, name: 'index_data_app_sessions_on_session_id'
+    add_index :data_app_sessions, :data_app_id, name: 'index_data_app_sessions_on_data_app_id'
+  end
+end
diff --git a/server/db/migrate/20241024103527_add_session_id_and_session_count_to_feedbacks_and_visual_components.rb b/server/db/migrate/20241024103527_add_session_id_and_session_count_to_feedbacks_and_visual_components.rb
new file mode 100644
index 00000000..784a05e9
--- /dev/null
+++ b/server/db/migrate/20241024103527_add_session_id_and_session_count_to_feedbacks_and_visual_components.rb
@@ -0,0 +1,11 @@
+class AddSessionIdAndSessionCountToFeedbacksAndVisualComponents < ActiveRecord::Migration[7.1]
+  def up
+    add_column :feedbacks, :session_id, :string, default: nil, null: true
+    add_column :visual_components, :session_count, :integer, default: 0
+  end
+
+  def down
+    remove_column :feedbacks, :session_id
+    remove_column :visual_components, :session_count
+  end
+end
diff --git a/server/db/schema.rb b/server/db/schema.rb
index 7e8b08c4..7de8dcfc 100644
--- a/server/db/schema.rb
+++ b/server/db/schema.rb
@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema[7.1].define(version: 2024_10_17_155337) do
+ActiveRecord::Schema[7.1].define(version: 2024_10_24_103527) do
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
 
@@ -48,6 +48,18 @@
     t.string "connector_category", default: "data", null: false
   end
 
+  create_table "data_app_sessions", force: :cascade do |t|
+    t.string "session_id", null: false
+    t.bigint "data_app_id", null: false
+    t.integer "workspace_id", null: false
+    t.datetime "start_time", null: false
+    t.datetime "end_time"
+    t.datetime "created_at", null: false
+    t.datetime "updated_at", null: false
+    t.index ["data_app_id"], name: "index_data_app_sessions_on_data_app_id"
+    t.index ["session_id"], name: "index_data_app_sessions_on_session_id", unique: true
+  end
+
   create_table "data_apps", force: :cascade do |t|
     t.string "name", null: false
     t.integer "status", null: false
@@ -73,6 +85,7 @@
     t.datetime "created_at", null: false
     t.datetime "updated_at", null: false
     t.integer "feedback_type", default: 0, null: false
+    t.string "session_id"
   end
 
   create_table "models", force: :cascade do |t|
@@ -220,6 +233,7 @@
     t.jsonb "feedback_config"
     t.datetime "created_at", null: false
     t.datetime "updated_at", null: false
+    t.integer "session_count", default: 0
   end
 
   create_table "workspace_users", force: :cascade do |t|
diff --git a/server/spec/factories/data_app_sessions.rb b/server/spec/factories/data_app_sessions.rb
new file mode 100644
index 00000000..e84f7fa5
--- /dev/null
+++ b/server/spec/factories/data_app_sessions.rb
@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+FactoryBot.define do
+  factory :data_app_session do
+    session_id { SecureRandom.hex(10) }
+    data_app
+    workspace
+  end
+end
diff --git a/server/spec/models/data_app_session_spec.rb b/server/spec/models/data_app_session_spec.rb
new file mode 100644
index 00000000..7943e38d
--- /dev/null
+++ b/server/spec/models/data_app_session_spec.rb
@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+require "rails_helper"
+
+RSpec.describe DataAppSession, type: :model do
+  let(:workspace) { create(:workspace) }
+  let(:data_app) { create(:data_app, workspace:) }
+
+  describe "associations" do
+    it { should belong_to(:data_app) }
+    it { should belong_to(:workspace) }
+  end
+
+  describe "validations" do
+    subject { build(:data_app_session, workspace:, data_app:) }
+
+    it { should validate_presence_of(:session_id) }
+    it { should validate_uniqueness_of(:session_id) }
+    it { should validate_presence_of(:data_app_id) }
+    it { should validate_presence_of(:workspace_id) }
+  end
+
+  describe "callbacks" do
+    it "sets the start_time and end_time before creation" do
+      session = DataAppSession.new(workspace:, data_app:, session_id: "session_abc")
+      session.save
+      expect(session.start_time).to be_present
+      expect(session.end_time).to eq(session.start_time + 10.minutes)
+    end
+  end
+
+  describe ".active scope" do
+    it "returns active sessions" do
+      active_session = create(:data_app_session, workspace:, data_app:)
+      expired_session = create(:data_app_session, workspace:, data_app:)
+      expired_session.update(end_time: 1.minute.ago)
+      expect(DataAppSession.active).to include(active_session)
+      expect(DataAppSession.active).not_to include(expired_session)
+    end
+  end
+
+  describe "#expired?" do
+    it "returns true if the session is expired" do
+      session = create(:data_app_session, workspace:, data_app:)
+      session.update(end_time: 1.minute.ago)
+      expect(session.expired?).to be true
+    end
+
+    it "returns false if the session is active" do
+      session = create(:data_app_session, workspace:, data_app:)
+      expect(session.expired?).to be false
+    end
+  end
+end
diff --git a/server/spec/models/data_app_spec.rb b/server/spec/models/data_app_spec.rb
index 1eee7dad..c863eadd 100644
--- a/server/spec/models/data_app_spec.rb
+++ b/server/spec/models/data_app_spec.rb
@@ -14,6 +14,7 @@
     it { should belong_to(:workspace) }
     it { should have_many(:visual_components).dependent(:destroy) }
     it { should have_many(:models).through(:visual_components) }
+    it { should have_many(:data_app_sessions).dependent(:nullify) }
   end
 
   describe "#set_default_status" do
diff --git a/server/spec/models/workspace_spec.rb b/server/spec/models/workspace_spec.rb
index 5e0e1ce3..6f118489 100644
--- a/server/spec/models/workspace_spec.rb
+++ b/server/spec/models/workspace_spec.rb
@@ -30,6 +30,7 @@
     it { should have_many(:catalogs).dependent(:nullify) }
     it { should have_many(:syncs).dependent(:nullify) }
     it { should have_many(:data_apps).dependent(:nullify) }
+    it { should have_many(:data_app_sessions).dependent(:nullify) }
     it { should belong_to(:organization) }
   end
 

From 45ee1724f3d5ec531345d28c8a3572d71dca2cc5 Mon Sep 17 00:00:00 2001
From: "github-actions[bot]"
 <41898282+github-actions[bot]@users.noreply.github.com>
Date: Mon, 28 Oct 2024 12:16:56 +0530
Subject: [PATCH 07/48] chore(CE): added audit log to workspace (#442)

Co-authored-by: TivonB-AI2 <124182151+TivonB-AI2@users.noreply.github.com>
---
 server/app/models/workspace.rb                |  1 +
 .../app/serializers/audit_logs_serializer.rb  |  2 +-
 .../controllers/concerns/audit_logger_spec.rb |  2 +
 server/spec/models/workspace_spec.rb          |  1 +
 .../api/v1/catalogs_controller_spec.rb        |  8 ++++
 .../api/v1/connector_definitions_spec.rb      | 22 ++++++++++
 .../api/v1/connectors_controller_spec.rb      | 42 +++++++++++++++++++
 .../requests/api/v1/models_controller_spec.rb | 32 ++++++++++++++
 .../api/v1/schedule_syncs_controller_spec.rb  |  4 ++
 .../requests/api/v1/syncs_controller_spec.rb  | 34 +++++++++++++++
 10 files changed, 147 insertions(+), 1 deletion(-)

diff --git a/server/app/models/workspace.rb b/server/app/models/workspace.rb
index 7e3a21e3..f628c752 100644
--- a/server/app/models/workspace.rb
+++ b/server/app/models/workspace.rb
@@ -22,6 +22,7 @@ class Workspace < ApplicationRecord
   has_many :sync_runs, dependent: :nullify
   has_many :data_apps, dependent: :nullify
   has_many :data_app_sessions, dependent: :nullify
+  has_many :audit_logs, dependent: :nullify
   belongs_to :organization
 
   STATUS_ACTIVE = "active"
diff --git a/server/app/serializers/audit_logs_serializer.rb b/server/app/serializers/audit_logs_serializer.rb
index 83c0f785..2f44abd1 100644
--- a/server/app/serializers/audit_logs_serializer.rb
+++ b/server/app/serializers/audit_logs_serializer.rb
@@ -3,5 +3,5 @@
 # app/serializers/audit_logs_serializer.rb
 class AuditLogsSerializer < ActiveModel::Serializer
   attributes :id, :user_id, :action, :resource_type, :resource_id, :resource, :workspace_id,
-             :metadata
+             :metadata, :created_at, :updated_at
 end
diff --git a/server/spec/controllers/concerns/audit_logger_spec.rb b/server/spec/controllers/concerns/audit_logger_spec.rb
index 7259e43c..bb937601 100644
--- a/server/spec/controllers/concerns/audit_logger_spec.rb
+++ b/server/spec/controllers/concerns/audit_logger_spec.rb
@@ -31,6 +31,8 @@
         expect(audit_log.resource).to eq(nil)
         expect(audit_log.workspace.id).to eq(workspace.id)
         expect(audit_log.metadata).to eq(payload.to_unsafe_h)
+        expect(audit_log.created_at).not_to be_nil
+        expect(audit_log.updated_at).not_to be_nil
       end
     end
 
diff --git a/server/spec/models/workspace_spec.rb b/server/spec/models/workspace_spec.rb
index 6f118489..1459dd76 100644
--- a/server/spec/models/workspace_spec.rb
+++ b/server/spec/models/workspace_spec.rb
@@ -31,6 +31,7 @@
     it { should have_many(:syncs).dependent(:nullify) }
     it { should have_many(:data_apps).dependent(:nullify) }
     it { should have_many(:data_app_sessions).dependent(:nullify) }
+    it { should have_many(:audit_logs).dependent(:nullify) }
     it { should belong_to(:organization) }
   end
 
diff --git a/server/spec/requests/api/v1/catalogs_controller_spec.rb b/server/spec/requests/api/v1/catalogs_controller_spec.rb
index 066e2728..ec7c7ead 100644
--- a/server/spec/requests/api/v1/catalogs_controller_spec.rb
+++ b/server/spec/requests/api/v1/catalogs_controller_spec.rb
@@ -96,6 +96,8 @@
         expect(audit_log.resource_id).to eq(nil)
         expect(audit_log.resource).to eq(connector.name)
         expect(audit_log.workspace_id).to eq(workspace.id)
+        expect(audit_log.created_at).not_to be_nil
+        expect(audit_log.updated_at).not_to be_nil
       end
     end
 
@@ -123,6 +125,8 @@
         expect(audit_log.resource_id).to eq(nil)
         expect(audit_log.resource).to eq(connector.name)
         expect(audit_log.workspace_id).to eq(workspace.id)
+        expect(audit_log.created_at).not_to be_nil
+        expect(audit_log.updated_at).not_to be_nil
       end
     end
 
@@ -172,6 +176,8 @@
         expect(audit_log.resource_id).to eq(existing_catalog.id)
         expect(audit_log.resource).to eq(connector.name)
         expect(audit_log.workspace_id).to eq(workspace.id)
+        expect(audit_log.created_at).not_to be_nil
+        expect(audit_log.updated_at).not_to be_nil
       end
     end
 
@@ -200,6 +206,8 @@
         expect(audit_log.resource_id).to eq(existing_catalog.id)
         expect(audit_log.resource).to eq(connector.name)
         expect(audit_log.workspace_id).to eq(workspace.id)
+        expect(audit_log.created_at).not_to be_nil
+        expect(audit_log.updated_at).not_to be_nil
       end
     end
 
diff --git a/server/spec/requests/api/v1/connector_definitions_spec.rb b/server/spec/requests/api/v1/connector_definitions_spec.rb
index d679b349..57a69c6d 100644
--- a/server/spec/requests/api/v1/connector_definitions_spec.rb
+++ b/server/spec/requests/api/v1/connector_definitions_spec.rb
@@ -40,6 +40,8 @@
         expect(audit_log.resource_id).to eq(nil)
         expect(audit_log.resource).to eq(nil)
         expect(audit_log.workspace_id).to eq(workspace.id)
+        expect(audit_log.created_at).not_to be_nil
+        expect(audit_log.updated_at).not_to be_nil
       end
 
       it "returns only ai/ml sources" do
@@ -63,6 +65,8 @@
         expect(audit_log.resource_id).to eq(nil)
         expect(audit_log.resource).to eq(nil)
         expect(audit_log.workspace_id).to eq(workspace.id)
+        expect(audit_log.created_at).not_to be_nil
+        expect(audit_log.updated_at).not_to be_nil
       end
 
       it "returns only ai/ml connectors" do
@@ -86,6 +90,8 @@
         expect(audit_log.resource_id).to eq(nil)
         expect(audit_log.resource).to eq(nil)
         expect(audit_log.workspace_id).to eq(workspace.id)
+        expect(audit_log.created_at).not_to be_nil
+        expect(audit_log.updated_at).not_to be_nil
       end
 
       it "returns success viewer role" do
@@ -105,6 +111,8 @@
         expect(audit_log.resource_id).to eq(nil)
         expect(audit_log.resource).to eq(nil)
         expect(audit_log.workspace_id).to eq(workspace.id)
+        expect(audit_log.created_at).not_to be_nil
+        expect(audit_log.updated_at).not_to be_nil
       end
 
       it "returns success member role" do
@@ -124,6 +132,8 @@
         expect(audit_log.resource_id).to eq(nil)
         expect(audit_log.resource).to eq(nil)
         expect(audit_log.workspace_id).to eq(workspace.id)
+        expect(audit_log.created_at).not_to be_nil
+        expect(audit_log.updated_at).not_to be_nil
       end
     end
   end
@@ -146,6 +156,8 @@
       expect(audit_log.resource_id).to eq(0)
       expect(audit_log.resource).to eq("Snowflake")
       expect(audit_log.workspace_id).to eq(workspace.id)
+      expect(audit_log.created_at).not_to be_nil
+      expect(audit_log.updated_at).not_to be_nil
     end
 
     it "returns a connector when found for member role" do
@@ -166,6 +178,8 @@
       expect(audit_log.resource_id).to eq(0)
       expect(audit_log.resource).to eq("Snowflake")
       expect(audit_log.workspace_id).to eq(workspace.id)
+      expect(audit_log.created_at).not_to be_nil
+      expect(audit_log.updated_at).not_to be_nil
     end
 
     it "returns a connector when found for viewer role" do
@@ -186,6 +200,8 @@
       expect(audit_log.resource_id).to eq(0)
       expect(audit_log.resource).to eq("Snowflake")
       expect(audit_log.workspace_id).to eq(workspace.id)
+      expect(audit_log.created_at).not_to be_nil
+      expect(audit_log.updated_at).not_to be_nil
     end
 
     it "returns empty array not found" do
@@ -204,6 +220,8 @@
       expect(audit_log.resource_id).to eq(0)
       expect(audit_log.resource).to eq(nil)
       expect(audit_log.workspace_id).to eq(workspace.id)
+      expect(audit_log.created_at).not_to be_nil
+      expect(audit_log.updated_at).not_to be_nil
     end
   end
 
@@ -239,6 +257,8 @@
       expect(audit_log.resource_id).to eq(nil)
       expect(audit_log.resource).to eq("Snowflake")
       expect(audit_log.workspace_id).to eq(workspace.id)
+      expect(audit_log.created_at).not_to be_nil
+      expect(audit_log.updated_at).not_to be_nil
     end
 
     it "returns success status for a valid connection fro member role" do
@@ -263,6 +283,8 @@
       expect(audit_log.resource_id).to eq(nil)
       expect(audit_log.resource).to eq("Snowflake")
       expect(audit_log.workspace_id).to eq(workspace.id)
+      expect(audit_log.created_at).not_to be_nil
+      expect(audit_log.updated_at).not_to be_nil
     end
 
     it "returns authorization failure for a view role user" do
diff --git a/server/spec/requests/api/v1/connectors_controller_spec.rb b/server/spec/requests/api/v1/connectors_controller_spec.rb
index 1c5b272a..c833a49e 100644
--- a/server/spec/requests/api/v1/connectors_controller_spec.rb
+++ b/server/spec/requests/api/v1/connectors_controller_spec.rb
@@ -48,6 +48,8 @@
         expect(audit_log.resource_id).to eq(nil)
         expect(audit_log.resource).to eq(nil)
         expect(audit_log.workspace_id).to eq(workspace.id)
+        expect(audit_log.created_at).not_to be_nil
+        expect(audit_log.updated_at).not_to be_nil
       end
 
       it "returns success and all connectors member role" do
@@ -67,6 +69,8 @@
         expect(audit_log.resource_id).to eq(nil)
         expect(audit_log.resource).to eq(nil)
         expect(audit_log.workspace_id).to eq(workspace.id)
+        expect(audit_log.created_at).not_to be_nil
+        expect(audit_log.updated_at).not_to be_nil
       end
 
       it "returns success and all connectors viewer role" do
@@ -96,6 +100,8 @@
         expect(audit_log.resource_id).to eq(nil)
         expect(audit_log.resource).to eq(nil)
         expect(audit_log.workspace_id).to eq(workspace.id)
+        expect(audit_log.created_at).not_to be_nil
+        expect(audit_log.updated_at).not_to be_nil
       end
 
       it "returns success and all source connectors" do
@@ -115,6 +121,8 @@
         expect(audit_log.resource_id).to eq(nil)
         expect(audit_log.resource).to eq(nil)
         expect(audit_log.workspace_id).to eq(workspace.id)
+        expect(audit_log.created_at).not_to be_nil
+        expect(audit_log.updated_at).not_to be_nil
       end
 
       it "returns success and destination connectors" do
@@ -134,6 +142,8 @@
         expect(audit_log.resource_id).to eq(nil)
         expect(audit_log.resource).to eq(nil)
         expect(audit_log.workspace_id).to eq(workspace.id)
+        expect(audit_log.created_at).not_to be_nil
+        expect(audit_log.updated_at).not_to be_nil
       end
 
       it "returns only data connectors" do
@@ -193,6 +203,8 @@
         expect(audit_log.resource_id).to eq(connectors.first.id)
         expect(audit_log.resource).to eq(connectors.first.name)
         expect(audit_log.workspace_id).to eq(workspace.id)
+        expect(audit_log.created_at).not_to be_nil
+        expect(audit_log.updated_at).not_to be_nil
       end
 
       it "returns success and fetch connector viewer role" do
@@ -230,6 +242,8 @@
         expect(audit_log.resource_id).to eq(connectors.first.id)
         expect(audit_log.resource).to eq(connectors.first.name)
         expect(audit_log.workspace_id).to eq(workspace.id)
+        expect(audit_log.created_at).not_to be_nil
+        expect(audit_log.updated_at).not_to be_nil
       end
 
       it "returns an error response while fetch connector" do
@@ -289,6 +303,8 @@
         expect(audit_log.resource_id).to eq(nil)
         expect(audit_log.resource).to eq(request_body.dig(:connector, :name))
         expect(audit_log.workspace_id).to eq(workspace.id)
+        expect(audit_log.created_at).not_to be_nil
+        expect(audit_log.updated_at).not_to be_nil
       end
 
       it "creates a new connector and returns success for member role" do
@@ -312,6 +328,8 @@
         expect(audit_log.resource_id).to eq(nil)
         expect(audit_log.resource).to eq(request_body.dig(:connector, :name))
         expect(audit_log.workspace_id).to eq(workspace.id)
+        expect(audit_log.created_at).not_to be_nil
+        expect(audit_log.updated_at).not_to be_nil
       end
 
       it "returns unauthorize viewer role" do
@@ -376,6 +394,8 @@
         expect(audit_log.resource_id).to eq(connectors.second.id)
         expect(audit_log.resource).to eq(request_body.dig(:connector, :name))
         expect(audit_log.workspace_id).to eq(workspace.id)
+        expect(audit_log.created_at).not_to be_nil
+        expect(audit_log.updated_at).not_to be_nil
       end
 
       it "updates the connector and returns success for member role" do
@@ -397,6 +417,8 @@
         expect(audit_log.resource_id).to eq(connectors.second.id)
         expect(audit_log.resource).to eq(request_body.dig(:connector, :name))
         expect(audit_log.workspace_id).to eq(workspace.id)
+        expect(audit_log.created_at).not_to be_nil
+        expect(audit_log.updated_at).not_to be_nil
       end
 
       it "returns unauthorize viewer role" do
@@ -443,6 +465,8 @@
         expect(audit_log.resource_id).to eq(connectors.first.id)
         expect(audit_log.resource).to eq(connectors.first.name)
         expect(audit_log.workspace_id).to eq(workspace.id)
+        expect(audit_log.created_at).not_to be_nil
+        expect(audit_log.updated_at).not_to be_nil
       end
 
       it "returns success and discover object for member role" do
@@ -465,6 +489,8 @@
         expect(audit_log.resource_id).to eq(connectors.first.id)
         expect(audit_log.resource).to eq(connectors.first.name)
         expect(audit_log.workspace_id).to eq(workspace.id)
+        expect(audit_log.created_at).not_to be_nil
+        expect(audit_log.updated_at).not_to be_nil
       end
 
       it "returns success and should not refresh the catalog when refresh flag is absent" do
@@ -486,6 +512,8 @@
         expect(audit_log.resource_id).to eq(connectors.first.id)
         expect(audit_log.resource).to eq(connectors.first.name)
         expect(audit_log.workspace_id).to eq(workspace.id)
+        expect(audit_log.created_at).not_to be_nil
+        expect(audit_log.updated_at).not_to be_nil
       end
 
       it "returns success and refresh the catalog when refresh is true" do
@@ -507,6 +535,8 @@
         expect(audit_log.resource_id).to eq(connectors.first.id)
         expect(audit_log.resource).to eq(connectors.first.name)
         expect(audit_log.workspace_id).to eq(workspace.id)
+        expect(audit_log.created_at).not_to be_nil
+        expect(audit_log.updated_at).not_to be_nil
       end
 
       it "returns success and discover object for viewer role" do
@@ -529,6 +559,8 @@
         expect(audit_log.resource_id).to eq(connectors.first.id)
         expect(audit_log.resource).to eq(connectors.first.name)
         expect(audit_log.workspace_id).to eq(workspace.id)
+        expect(audit_log.created_at).not_to be_nil
+        expect(audit_log.updated_at).not_to be_nil
       end
 
       it "returns an error response while get discover object" do
@@ -561,6 +593,8 @@
         expect(audit_log.resource_id).to eq(connectors.first.id)
         expect(audit_log.resource).to eq(connectors.first.name)
         expect(audit_log.workspace_id).to eq(workspace.id)
+        expect(audit_log.created_at).not_to be_nil
+        expect(audit_log.updated_at).not_to be_nil
       end
 
       it "returns success and delete connector" do
@@ -576,6 +610,8 @@
         expect(audit_log.resource_id).to eq(connectors.first.id)
         expect(audit_log.resource).to eq(connectors.first.name)
         expect(audit_log.workspace_id).to eq(workspace.id)
+        expect(audit_log.created_at).not_to be_nil
+        expect(audit_log.updated_at).not_to be_nil
       end
 
       it "returns fail viwer role" do
@@ -639,6 +675,8 @@
         expect(audit_log.resource_id).to eq(connector.id)
         expect(audit_log.resource).to eq(connector.name)
         expect(audit_log.workspace_id).to eq(workspace.id)
+        expect(audit_log.created_at).not_to be_nil
+        expect(audit_log.updated_at).not_to be_nil
       end
 
       it "returns success status for a valid query for member role" do
@@ -659,6 +697,8 @@
         expect(audit_log.resource_id).to eq(connector.id)
         expect(audit_log.resource).to eq(connector.name)
         expect(audit_log.workspace_id).to eq(workspace.id)
+        expect(audit_log.created_at).not_to be_nil
+        expect(audit_log.updated_at).not_to be_nil
       end
 
       it "returns an error message for missing catalog for ai connectors" do
@@ -713,6 +753,8 @@
         expect(audit_log.resource_id).to eq(connector.id)
         expect(audit_log.resource).to eq(connector.name)
         expect(audit_log.workspace_id).to eq(workspace.id)
+        expect(audit_log.created_at).not_to be_nil
+        expect(audit_log.updated_at).not_to be_nil
       end
 
       it "returns failure status for a invalid query" do
diff --git a/server/spec/requests/api/v1/models_controller_spec.rb b/server/spec/requests/api/v1/models_controller_spec.rb
index 494c1497..7cb73a4f 100644
--- a/server/spec/requests/api/v1/models_controller_spec.rb
+++ b/server/spec/requests/api/v1/models_controller_spec.rb
@@ -59,6 +59,8 @@
         expect(audit_log.resource_id).to eq(nil)
         expect(audit_log.resource).to eq(nil)
         expect(audit_log.workspace_id).to eq(workspace.id)
+        expect(audit_log.created_at).not_to be_nil
+        expect(audit_log.updated_at).not_to be_nil
       end
 
       it "returns success and all mode for viewer role" do
@@ -88,6 +90,8 @@
         expect(audit_log.resource_id).to eq(nil)
         expect(audit_log.resource).to eq(nil)
         expect(audit_log.workspace_id).to eq(workspace.id)
+        expect(audit_log.created_at).not_to be_nil
+        expect(audit_log.updated_at).not_to be_nil
       end
 
       it "filters models based on the query_type parameter" do
@@ -103,6 +107,8 @@
         expect(audit_log.resource_id).to eq(nil)
         expect(audit_log.resource).to eq(nil)
         expect(audit_log.workspace_id).to eq(workspace.id)
+        expect(audit_log.created_at).not_to be_nil
+        expect(audit_log.updated_at).not_to be_nil
       end
 
       it "filters models based on a different query_type" do
@@ -120,6 +126,8 @@
         expect(audit_log.resource_id).to eq(nil)
         expect(audit_log.resource).to eq(nil)
         expect(audit_log.workspace_id).to eq(workspace.id)
+        expect(audit_log.created_at).not_to be_nil
+        expect(audit_log.updated_at).not_to be_nil
       end
 
       it "returns all models" do
@@ -136,6 +144,8 @@
         expect(audit_log.resource_id).to eq(nil)
         expect(audit_log.resource).to eq(nil)
         expect(audit_log.workspace_id).to eq(workspace.id)
+        expect(audit_log.created_at).not_to be_nil
+        expect(audit_log.updated_at).not_to be_nil
       end
     end
   end
@@ -169,6 +179,8 @@
         expect(audit_log.resource_id).to eq(models.first.id)
         expect(audit_log.resource).to eq(models.first.name)
         expect(audit_log.workspace_id).to eq(workspace.id)
+        expect(audit_log.created_at).not_to be_nil
+        expect(audit_log.updated_at).not_to be_nil
       end
 
       it "returns success and fetch model for viewer role" do
@@ -206,6 +218,8 @@
         expect(audit_log.resource_id).to eq(models.first.id)
         expect(audit_log.resource).to eq(models.first.name)
         expect(audit_log.workspace_id).to eq(workspace.id)
+        expect(audit_log.created_at).not_to be_nil
+        expect(audit_log.updated_at).not_to be_nil
       end
 
       it "returns an error response while fetch model" do
@@ -255,6 +269,8 @@
         expect(audit_log.resource_id).to eq(nil)
         expect(audit_log.resource).to eq(request_body.dig(:model, :name))
         expect(audit_log.workspace_id).to eq(workspace.id)
+        expect(audit_log.created_at).not_to be_nil
+        expect(audit_log.updated_at).not_to be_nil
       end
 
       it "fails model creation for connector without catalog" do
@@ -297,6 +313,8 @@
         expect(audit_log.resource_id).to eq(nil)
         expect(audit_log.resource).to eq(request_body.dig(:model, :name))
         expect(audit_log.workspace_id).to eq(workspace.id)
+        expect(audit_log.created_at).not_to be_nil
+        expect(audit_log.updated_at).not_to be_nil
       end
 
       it "creates a new model with query type table selector" do
@@ -321,6 +339,8 @@
         expect(audit_log.resource_id).to eq(nil)
         expect(audit_log.resource).to eq(request_body.dig(:model, :name))
         expect(audit_log.workspace_id).to eq(workspace.id)
+        expect(audit_log.created_at).not_to be_nil
+        expect(audit_log.updated_at).not_to be_nil
       end
 
       context "when creating a model with query_type = ai_ml and configuration is present" do
@@ -354,6 +374,8 @@
           expect(audit_log.resource_id).to eq(nil)
           expect(audit_log.resource).to eq(request_body.dig(:model, :name))
           expect(audit_log.workspace_id).to eq(workspace.id)
+          expect(audit_log.created_at).not_to be_nil
+          expect(audit_log.updated_at).not_to be_nil
         end
       end
 
@@ -414,6 +436,8 @@
         expect(audit_log.resource_id).to eq(models.second.id)
         expect(audit_log.resource).to eq(request_body.dig(:model, :name))
         expect(audit_log.workspace_id).to eq(workspace.id)
+        expect(audit_log.created_at).not_to be_nil
+        expect(audit_log.updated_at).not_to be_nil
       end
 
       it "fails model update for connector without catalog for ai model" do
@@ -463,6 +487,8 @@
         expect(audit_log.resource_id).to eq(models.second.id)
         expect(audit_log.resource).to eq(request_body.dig(:model, :name))
         expect(audit_log.workspace_id).to eq(workspace.id)
+        expect(audit_log.created_at).not_to be_nil
+        expect(audit_log.updated_at).not_to be_nil
       end
 
       context "when updating a model with query_type = ai_ml and configuration is present" do
@@ -496,6 +522,8 @@
           expect(audit_log.resource_id).to eq(models.second.id)
           expect(audit_log.resource).to eq(request_body.dig(:model, :name))
           expect(audit_log.workspace_id).to eq(workspace.id)
+          expect(audit_log.created_at).not_to be_nil
+          expect(audit_log.updated_at).not_to be_nil
         end
       end
 
@@ -542,6 +570,8 @@
         expect(audit_log.resource_id).to eq(models.first.id)
         expect(audit_log.resource).to eq(models.first.name)
         expect(audit_log.workspace_id).to eq(workspace.id)
+        expect(audit_log.created_at).not_to be_nil
+        expect(audit_log.updated_at).not_to be_nil
       end
 
       it "returns success and delete model for member role" do
@@ -557,6 +587,8 @@
         expect(audit_log.resource_id).to eq(models.first.id)
         expect(audit_log.resource).to eq(models.first.name)
         expect(audit_log.workspace_id).to eq(workspace.id)
+        expect(audit_log.created_at).not_to be_nil
+        expect(audit_log.updated_at).not_to be_nil
       end
 
       it "returns for viwer role " do
diff --git a/server/spec/requests/api/v1/schedule_syncs_controller_spec.rb b/server/spec/requests/api/v1/schedule_syncs_controller_spec.rb
index 2b555887..8bee8391 100644
--- a/server/spec/requests/api/v1/schedule_syncs_controller_spec.rb
+++ b/server/spec/requests/api/v1/schedule_syncs_controller_spec.rb
@@ -67,6 +67,8 @@
         expect(audit_log.resource_id).to eq(nil)
         expect(audit_log.resource).to eq(request_body.dig(:sync, :name))
         expect(audit_log.workspace_id).to eq(workspace.id)
+        expect(audit_log.created_at).not_to be_nil
+        expect(audit_log.updated_at).not_to be_nil
       end
     end
 
@@ -156,6 +158,8 @@
         expect(audit_log.resource_id).to eq(nil)
         expect(audit_log.resource).to eq(request_body.dig(:sync, :name))
         expect(audit_log.workspace_id).to eq(workspace.id)
+        expect(audit_log.created_at).not_to be_nil
+        expect(audit_log.updated_at).not_to be_nil
       end
     end
 
diff --git a/server/spec/requests/api/v1/syncs_controller_spec.rb b/server/spec/requests/api/v1/syncs_controller_spec.rb
index 2bb225ee..eab61679 100644
--- a/server/spec/requests/api/v1/syncs_controller_spec.rb
+++ b/server/spec/requests/api/v1/syncs_controller_spec.rb
@@ -65,6 +65,8 @@
         expect(audit_log.resource_id).to eq(nil)
         expect(audit_log.resource).to eq(nil)
         expect(audit_log.workspace_id).to eq(workspace.id)
+        expect(audit_log.created_at).not_to be_nil
+        expect(audit_log.updated_at).not_to be_nil
       end
 
       it "returns success and get all syncs for member role" do
@@ -90,6 +92,8 @@
         expect(audit_log.resource_id).to eq(nil)
         expect(audit_log.resource).to eq(nil)
         expect(audit_log.workspace_id).to eq(workspace.id)
+        expect(audit_log.created_at).not_to be_nil
+        expect(audit_log.updated_at).not_to be_nil
       end
 
       it "returns success and get all syncs for viewer role" do
@@ -145,6 +149,8 @@
         expect(audit_log.resource_id).to eq(syncs.first.id)
         expect(audit_log.resource).to eq(nil)
         expect(audit_log.workspace_id).to eq(workspace.id)
+        expect(audit_log.created_at).not_to be_nil
+        expect(audit_log.updated_at).not_to be_nil
       end
 
       it "returns success and fetch sync for viewer_role" do
@@ -174,6 +180,8 @@
         expect(audit_log.resource_id).to eq(syncs.first.id)
         expect(audit_log.resource).to eq(nil)
         expect(audit_log.workspace_id).to eq(workspace.id)
+        expect(audit_log.created_at).not_to be_nil
+        expect(audit_log.updated_at).not_to be_nil
       end
 
       it "returns success and fetch sync for member_role" do
@@ -203,6 +211,8 @@
         expect(audit_log.resource_id).to eq(syncs.first.id)
         expect(audit_log.resource).to eq(nil)
         expect(audit_log.workspace_id).to eq(workspace.id)
+        expect(audit_log.created_at).not_to be_nil
+        expect(audit_log.updated_at).not_to be_nil
       end
 
       it "returns an error response while fetch sync" do
@@ -270,6 +280,8 @@
         expect(audit_log.resource_id).to eq(nil)
         expect(audit_log.resource).to eq(request_body.dig(:sync, :name))
         expect(audit_log.workspace_id).to eq(workspace.id)
+        expect(audit_log.created_at).not_to be_nil
+        expect(audit_log.updated_at).not_to be_nil
       end
 
       it "creates a new sync and returns success with cursor_field nil " do
@@ -297,6 +309,8 @@
         expect(audit_log.resource_id).to eq(nil)
         expect(audit_log.resource).to eq(request_body.dig(:sync, :name))
         expect(audit_log.workspace_id).to eq(workspace.id)
+        expect(audit_log.created_at).not_to be_nil
+        expect(audit_log.updated_at).not_to be_nil
       end
 
       it "creates a new sync and returns success with cursor_field nil for member role" do
@@ -325,6 +339,8 @@
         expect(audit_log.resource_id).to eq(nil)
         expect(audit_log.resource).to eq(request_body.dig(:sync, :name))
         expect(audit_log.workspace_id).to eq(workspace.id)
+        expect(audit_log.created_at).not_to be_nil
+        expect(audit_log.updated_at).not_to be_nil
       end
 
       it "creates a new sync and returns unauthorized for viewer role" do
@@ -384,6 +400,8 @@
         expect(audit_log.resource_id).to eq(nil)
         expect(audit_log.resource).to eq(request_body.dig(:sync, :name))
         expect(audit_log.workspace_id).to eq(workspace.id)
+        expect(audit_log.created_at).not_to be_nil
+        expect(audit_log.updated_at).not_to be_nil
       end
 
       context "when  schedule type is cron expression " do
@@ -408,6 +426,8 @@
           expect(audit_log.resource_id).to eq(nil)
           expect(audit_log.resource).to eq(request_body.dig(:sync, :name))
           expect(audit_log.workspace_id).to eq(workspace.id)
+          expect(audit_log.created_at).not_to be_nil
+          expect(audit_log.updated_at).not_to be_nil
         end
       end
     end
@@ -462,6 +482,8 @@
         expect(audit_log.resource_id).to eq(syncs.first.id)
         expect(audit_log.resource).to eq(request_body.dig(:sync, :name))
         expect(audit_log.workspace_id).to eq(workspace.id)
+        expect(audit_log.created_at).not_to be_nil
+        expect(audit_log.updated_at).not_to be_nil
       end
 
       it "updates the sync and returns success for member role" do
@@ -487,6 +509,8 @@
         expect(audit_log.resource_id).to eq(syncs.first.id)
         expect(audit_log.resource).to eq(request_body.dig(:sync, :name))
         expect(audit_log.workspace_id).to eq(workspace.id)
+        expect(audit_log.created_at).not_to be_nil
+        expect(audit_log.updated_at).not_to be_nil
       end
 
       it "updates the sync and returns success for viewer role" do
@@ -540,6 +564,8 @@
         expect(audit_log.resource_id).to eq(syncs.first.id)
         expect(audit_log.resource).to eq(request_body.dig(:sync, :name))
         expect(audit_log.workspace_id).to eq(workspace.id)
+        expect(audit_log.created_at).not_to be_nil
+        expect(audit_log.updated_at).not_to be_nil
       end
 
       context "when  schedule type is cron expression " do
@@ -564,6 +590,8 @@
           expect(audit_log.resource_id).to eq(syncs.first.id)
           expect(audit_log.resource).to eq(request_body.dig(:sync, :name))
           expect(audit_log.workspace_id).to eq(workspace.id)
+          expect(audit_log.created_at).not_to be_nil
+          expect(audit_log.updated_at).not_to be_nil
         end
       end
     end
@@ -608,6 +636,8 @@
         expect(audit_log.resource_id).to eq(syncs.first.id)
         expect(audit_log.resource).to eq(request_body.dig(:sync, :name))
         expect(audit_log.workspace_id).to eq(workspace.id)
+        expect(audit_log.created_at).not_to be_nil
+        expect(audit_log.updated_at).not_to be_nil
       end
     end
   end
@@ -664,6 +694,8 @@
         expect(audit_log.resource_id).to eq(syncs.first.id)
         expect(audit_log.resource).to eq(nil)
         expect(audit_log.workspace_id).to eq(workspace.id)
+        expect(audit_log.created_at).not_to be_nil
+        expect(audit_log.updated_at).not_to be_nil
       end
 
       it "returns success and delete sync for member role" do
@@ -679,6 +711,8 @@
         expect(audit_log.resource_id).to eq(syncs.first.id)
         expect(audit_log.resource).to eq(nil)
         expect(audit_log.workspace_id).to eq(workspace.id)
+        expect(audit_log.created_at).not_to be_nil
+        expect(audit_log.updated_at).not_to be_nil
       end
 
       it "returns success and delete sync for viewer role" do

From d2348170de1322d158e09e6b306bd2026490f4fd Mon Sep 17 00:00:00 2001
From: "github-actions[bot]"
 <41898282+github-actions[bot]@users.noreply.github.com>
Date: Mon, 28 Oct 2024 12:17:28 +0530
Subject: [PATCH 08/48] chore(CE): add audit logs to factory and roles (#441)

Co-authored-by: TivonB-AI2 <124182151+TivonB-AI2@users.noreply.github.com>
---
 server/spec/factories/audit_logs.rb | 16 ++++++++++++++++
 server/spec/factories/roles.rb      |  9 ++++++---
 2 files changed, 22 insertions(+), 3 deletions(-)
 create mode 100644 server/spec/factories/audit_logs.rb

diff --git a/server/spec/factories/audit_logs.rb b/server/spec/factories/audit_logs.rb
new file mode 100644
index 00000000..e63ceced
--- /dev/null
+++ b/server/spec/factories/audit_logs.rb
@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+FactoryBot.define do
+  factory :audit_log do
+    action { "show" }
+    resource_type { "Sync" }
+    resource_id { 1 }
+    resource { "Test_Sync" }
+    metadata { nil }
+    created_at { Time.current }
+    updated_at { Time.current }
+
+    association :workspace
+    association :user
+  end
+end
diff --git a/server/spec/factories/roles.rb b/server/spec/factories/roles.rb
index 486b4370..4c6a2ae4 100644
--- a/server/spec/factories/roles.rb
+++ b/server/spec/factories/roles.rb
@@ -31,7 +31,8 @@
             "workspace" => { "read" => true, "create" => true, "delete" => true, "update" => true },
             "sync_record" => { "read" => true, "create" => true, "delete" => true, "update" => true },
             "connector_definition" => { "read" => true, "create" => true, "delete" => true, "update" => true },
-            "data_app" => { "read" => true, "create" => true, "delete" => true, "update" => true }
+            "data_app" => { "read" => true, "create" => true, "delete" => true, "update" => true },
+            "audit_logs" => { "read" => true, "create" => true, "delete" => true, "update" => true }
           }
         }
       end
@@ -52,7 +53,8 @@
             "workspace" => { "read" => true, "create" => false, "delete" => false, "update" => false },
             "sync_record" => { "read" => true, "create" => true, "delete" => true, "update" => true },
             "connector_definition" => { "read" => true, "create" => true, "delete" => true, "update" => true },
-            "data_app" => { "read" => true, "create" => true, "delete" => true, "update" => true }
+            "data_app" => { "read" => true, "create" => true, "delete" => true, "update" => true },
+            "audit_logs" => { "read" => true, "create" => true, "delete" => true, "update" => true }
           }
         }
       end
@@ -73,7 +75,8 @@
             "workspace" => { "read" => true, "create" => false, "delete" => false, "update" => false },
             "sync_record" => { "read" => true, "create" => false, "delete" => false, "update" => false },
             "connector_definition" => { "read" => true, "create" => false, "delete" => false, "update" => false },
-            "data_app" => { "read" => true, "create" => false, "delete" => false, "update" => false }
+            "data_app" => { "read" => true, "create" => false, "delete" => false, "update" => false },
+            "audit_logs" => { "read" => false, "create" => false, "delete" => false, "update" => false }
           }
         }
       end

From 98b04771649ec170029a75a3c835f0a9555b8cc3 Mon Sep 17 00:00:00 2001
From: "Rafael E. O'Neill"
 <106079170+RafaelOAiSquared@users.noreply.github.com>
Date: Mon, 28 Oct 2024 09:10:40 -0400
Subject: [PATCH 09/48] Multiwoven release v0.24.0 (#427)

Co-authored-by: github-actions <developer@multiwoven.com>
---
 release-notes.md | 35 ++++++++++++++++++-----------------
 1 file changed, 18 insertions(+), 17 deletions(-)

diff --git a/release-notes.md b/release-notes.md
index 868d77d0..385e5465 100644
--- a/release-notes.md
+++ b/release-notes.md
@@ -2,34 +2,35 @@
 
 All notable changes to this project will be documented in this file.
 
-## [0.23.0] - 2024-09-02
+## [0.24.0] - 2024-09-09
 
 ### 🚀 Features
 
-- *(CE)* Filter connectors based on the category (#338)
-- *(CE)* Filter connectors (#343)
-- *(CE)* Filter model based on the query type (#347)
-- *(CE)* Add aws sagemaker model source connector (#352)
+- *(CE)* Data app model changes (#353)
+- *(CE)* Verify user after signup
+- *(CE)* Enable and Disable sync via UI 
+- Added workflows for auto-deployment (#361)
 
 ### 🐛 Bug Fixes
 
-- *(CE)* Refresh catalog only when refresh is true (#336)
-- *(CE)* Connection_spec mandatory changes (#342)
-- *(CE)* Added tag label to auto-truncate text
-- *(CE)* Request log redaction (#348)
-- *(CE)* Add more fields to log redaction (#349)
+- *(CE)* Validate_catalog disabled query_source (#356)
+- *(CE)* Enable catalog validation only for ai models (#355)
+- *(CE)* Disable catalog validation for data models (#358)
+- *(CE)* Fixed sync runs on click function
+- *(CE)* Added try catch to model query preview API Call
+- *(CE)* Fixed sync mapping model column values
+- *(CE)* Test connection text
+- Adding more labels to self-hosted runner (#363)
 
 ### 🚜 Refactor
 
-- *(CE)* Created common connector lists component
+- *(CE)* Moved common elements of sign up and sign in to separate views
+- *(CE)* Changed Sync Records UX
+- *(CE)* Changed setup models to setup define
 
 ### ⚙️ Miscellaneous Tasks
 
-- *(CE)* Validate catalog for query source (#340)
-- *(CE)* Add catalog presence validation for models (#341)
-- *(CE)* Refactor interactors (#344)
-- *(CE)* Add filtering scope to connectors (#345)
-- *(CE)* Gem server update 0.10.0 (#351)
-- *(CE)* Data app model migration (#350)
+- *(CE)* Add data-app permissions to roles (#357)
+- *(CE)* Update sagemaker icon (#359)
 
 <!-- generated by git-cliff -->

From ca5b2b50d94d1f528dfc041d07bff00a64f55068 Mon Sep 17 00:00:00 2001
From: "Rafael E. O'Neill"
 <106079170+RafaelOAiSquared@users.noreply.github.com>
Date: Mon, 28 Oct 2024 09:12:36 -0400
Subject: [PATCH 10/48] Multiwoven release v0.25.0 (#428)

Co-authored-by: github-actions <developer@multiwoven.com>
Co-authored-by: Tushar Selvakumar <54372016+macintushar@users.noreply.github.com>
---
 release-notes.md | 31 ++++++++++---------------------
 1 file changed, 10 insertions(+), 21 deletions(-)

diff --git a/release-notes.md b/release-notes.md
index 385e5465..126aa59a 100644
--- a/release-notes.md
+++ b/release-notes.md
@@ -2,35 +2,24 @@
 
 All notable changes to this project will be documented in this file.
 
-## [0.24.0] - 2024-09-09
+## [0.25.0] - 2024-09-16
 
 ### 🚀 Features
 
-- *(CE)* Data app model changes (#353)
-- *(CE)* Verify user after signup
-- *(CE)* Enable and Disable sync via UI 
-- Added workflows for auto-deployment (#361)
+- *(CE)* Add google vertex model source connector (#365)
 
 ### 🐛 Bug Fixes
 
-- *(CE)* Validate_catalog disabled query_source (#356)
-- *(CE)* Enable catalog validation only for ai models (#355)
-- *(CE)* Disable catalog validation for data models (#358)
-- *(CE)* Fixed sync runs on click function
-- *(CE)* Added try catch to model query preview API Call
-- *(CE)* Fixed sync mapping model column values
-- *(CE)* Test connection text
-- Adding more labels to self-hosted runner (#363)
-
-### 🚜 Refactor
-
-- *(CE)* Moved common elements of sign up and sign in to separate views
-- *(CE)* Changed Sync Records UX
-- *(CE)* Changed setup models to setup define
+- *(CE)* Change clickhouse host to url (#370)
+- *(CE)* Fix weekly syncs bug (#369)
+- *(CE)* Change location to region in vertex (#372)
 
 ### ⚙️ Miscellaneous Tasks
 
-- *(CE)* Add data-app permissions to roles (#357)
-- *(CE)* Update sagemaker icon (#359)
+- *(CE)* Update server gem (#364)
+- *(CE)* Added x-request/response-format to spec (#368)
+- *(CE)* Update databricks json schema (#367)
+- *(CE)* Upgrade gem 0.11.3 (#366)
+- *(CE)* Update server gem 0.11.5 (#371)
 
 <!-- generated by git-cliff -->

From 383365c1c7899d70bb56c8056765f6f176fc883e Mon Sep 17 00:00:00 2001
From: "Rafael E. O'Neill"
 <106079170+RafaelOAiSquared@users.noreply.github.com>
Date: Mon, 28 Oct 2024 09:13:31 -0400
Subject: [PATCH 11/48] Multiwoven release v0.27.0 (#430)

Co-authored-by: github-actions <developer@multiwoven.com>
Co-authored-by: Tushar Selvakumar <54372016+macintushar@users.noreply.github.com>
---
 release-notes.md | 18 ++----------------
 1 file changed, 2 insertions(+), 16 deletions(-)

diff --git a/release-notes.md b/release-notes.md
index 126aa59a..bc637ae7 100644
--- a/release-notes.md
+++ b/release-notes.md
@@ -2,24 +2,10 @@
 
 All notable changes to this project will be documented in this file.
 
-## [0.25.0] - 2024-09-16
-
-### 🚀 Features
-
-- *(CE)* Add google vertex model source connector (#365)
-
-### 🐛 Bug Fixes
-
-- *(CE)* Change clickhouse host to url (#370)
-- *(CE)* Fix weekly syncs bug (#369)
-- *(CE)* Change location to region in vertex (#372)
+## [0.27.0] - 2024-09-27
 
 ### ⚙️ Miscellaneous Tasks
 
-- *(CE)* Update server gem (#364)
-- *(CE)* Added x-request/response-format to spec (#368)
-- *(CE)* Update databricks json schema (#367)
-- *(CE)* Upgrade gem 0.11.3 (#366)
-- *(CE)* Update server gem 0.11.5 (#371)
+- *(CE)* Prevent duplicate names for source connectors (#382)
 
 <!-- generated by git-cliff -->

From 818cba79c34d2f8dca9b39831fa7d5be76cf4aeb Mon Sep 17 00:00:00 2001
From: "Rafael E. O'Neill"
 <106079170+RafaelOAiSquared@users.noreply.github.com>
Date: Mon, 28 Oct 2024 09:14:14 -0400
Subject: [PATCH 12/48] Multiwoven release v0.28.0 (#431)

Co-authored-by: github-actions <developer@multiwoven.com>
Co-authored-by: Tushar Selvakumar <54372016+macintushar@users.noreply.github.com>
---
 release-notes.md | 16 ++++++++++++++--
 1 file changed, 14 insertions(+), 2 deletions(-)

diff --git a/release-notes.md b/release-notes.md
index bc637ae7..d1400544 100644
--- a/release-notes.md
+++ b/release-notes.md
@@ -2,10 +2,22 @@
 
 All notable changes to this project will be documented in this file.
 
-## [0.27.0] - 2024-09-27
+## [0.28.0] - 2024-10-07
+
+### 🚀 Features
+
+- *(CE)* Add ms sql connector (#456)
+
+### 🐛 Bug Fixes
+
+- *(CE)* Remove conditional init of temporal service (#397)
+- *(CE)* Endpoint mismatch in spec (#398)
+- *(CE)* Fixed duration units in sync runs
 
 ### ⚙️ Miscellaneous Tasks
 
-- *(CE)* Prevent duplicate names for source connectors (#382)
+- *(CE)* Add data_app_token with unique constraint (#379)
+- *(CE)* Add data_app_token generation (#381)
+- *(CE)* Gem update 0.11.6 (#399)
 
 <!-- generated by git-cliff -->

From c10c6378dc4f8260c0ad457a85a345695d025554 Mon Sep 17 00:00:00 2001
From: "Rafael E. O'Neill"
 <106079170+RafaelOAiSquared@users.noreply.github.com>
Date: Mon, 28 Oct 2024 09:14:47 -0400
Subject: [PATCH 13/48] Multiwoven release v0.29.0 (#432)

Co-authored-by: github-actions <developer@multiwoven.com>
Co-authored-by: Tushar Selvakumar <54372016+macintushar@users.noreply.github.com>
---
 release-notes.md | 13 +++++--------
 1 file changed, 5 insertions(+), 8 deletions(-)

diff --git a/release-notes.md b/release-notes.md
index d1400544..42e20d72 100644
--- a/release-notes.md
+++ b/release-notes.md
@@ -2,22 +2,19 @@
 
 All notable changes to this project will be documented in this file.
 
-## [0.28.0] - 2024-10-07
+## [0.29.0] - 2024-10-14
 
 ### 🚀 Features
 
-- *(CE)* Add ms sql connector (#456)
+- *(CE)* Added UTC as timezone for cron schedule
 
 ### 🐛 Bug Fixes
 
-- *(CE)* Remove conditional init of temporal service (#397)
-- *(CE)* Endpoint mismatch in spec (#398)
-- *(CE)* Fixed duration units in sync runs
+- *(CE)* Remove unique validation of organization name (#477) (#407)
 
 ### ⚙️ Miscellaneous Tasks
 
-- *(CE)* Add data_app_token with unique constraint (#379)
-- *(CE)* Add data_app_token generation (#381)
-- *(CE)* Gem update 0.11.6 (#399)
+- *(CE)* Add feedback model (#403)
+- *(CE)* Update sign up and reset password text (#406)
 
 <!-- generated by git-cliff -->

From c109963f224c4afebef4d95e8ab8ffd1cedcdab5 Mon Sep 17 00:00:00 2001
From: "Rafael E. O'Neill"
 <106079170+RafaelOAiSquared@users.noreply.github.com>
Date: Mon, 28 Oct 2024 09:15:24 -0400
Subject: [PATCH 14/48] Multiwoven release v0.30.0 (#433)

Co-authored-by: github-actions <developer@multiwoven.com>
Co-authored-by: Tushar Selvakumar <54372016+macintushar@users.noreply.github.com>
---
 release-notes.md | 18 ++++++++++--------
 1 file changed, 10 insertions(+), 8 deletions(-)

diff --git a/release-notes.md b/release-notes.md
index 42e20d72..7ae8dc6d 100644
--- a/release-notes.md
+++ b/release-notes.md
@@ -2,19 +2,21 @@
 
 All notable changes to this project will be documented in this file.
 
-## [0.29.0] - 2024-10-14
-
-### 🚀 Features
-
-- *(CE)* Added UTC as timezone for cron schedule
+## [0.30.0] - 2024-10-21
 
 ### 🐛 Bug Fixes
 
-- *(CE)* Remove unique validation of organization name (#477) (#407)
+- *(CE)* Server ci set ubuntu to 22.04 (#411)
+- *(CE)* Disable the database middle ware for temporal (#413)
+- *(CE)* Dashboard sidebar responsiveness
+- *(CE)* Wrong argument type NoMethodError (expected String) (#415)
 
 ### ⚙️ Miscellaneous Tasks
 
-- *(CE)* Add feedback model (#403)
-- *(CE)* Update sign up and reset password text (#406)
+- *(CE)* Configure user email verification based on env variable (#409)
+- *(CE)* Audit Model(DB) Changes (#416)
+- *(CE)* Feedback model changes (#414)
+- *(CE)* Reaction field enum correction (#491) (#418)
+- *(CE)* Add audit logger and audit to syncs controller (#417)
 
 <!-- generated by git-cliff -->

From 9b3634b71f4fd22f14b87bf28789bb84385a602a Mon Sep 17 00:00:00 2001
From: "github-actions[bot]"
 <41898282+github-actions[bot]@users.noreply.github.com>
Date: Tue, 29 Oct 2024 09:45:26 +0530
Subject: [PATCH 15/48] chore(CE): add audit logs to report controller (#446)

Co-authored-by: TivonB-AI2 <124182151+TivonB-AI2@users.noreply.github.com>
---
 .../controllers/api/v1/reports_controller.rb  |  6 ++++
 .../api/v1/reports_controller_spec.rb         | 33 +++++++++++++++++++
 2 files changed, 39 insertions(+)

diff --git a/server/app/controllers/api/v1/reports_controller.rb b/server/app/controllers/api/v1/reports_controller.rb
index 334d6ef4..8b4410c6 100644
--- a/server/app/controllers/api/v1/reports_controller.rb
+++ b/server/app/controllers/api/v1/reports_controller.rb
@@ -4,6 +4,8 @@ module Api
   module V1
     class ReportsController < ApplicationController
       include Reports
+      include AuditLogger
+      after_action :create_audit_log
       attr_reader :report
 
       def index
@@ -19,6 +21,10 @@ def index
 
       private
 
+      def create_audit_log
+        audit!
+      end
+
       def report_params
         params.permit(:type, :time_period, :metric, connector_ids: []).merge(workspace: current_workspace)
       end
diff --git a/server/spec/requests/api/v1/reports_controller_spec.rb b/server/spec/requests/api/v1/reports_controller_spec.rb
index e6e5e114..0e927cc3 100644
--- a/server/spec/requests/api/v1/reports_controller_spec.rb
+++ b/server/spec/requests/api/v1/reports_controller_spec.rb
@@ -73,6 +73,17 @@
         expect(total_sync_run_rows[slice_size - 1]["total_count"]).to eq(6)
         expect(total_sync_run_rows[slice_size - 1]["success_count"]).to eq(3)
         expect(total_sync_run_rows[slice_size - 1]["failed_count"]).to eq(3)
+
+        audit_log = AuditLog.last
+        expect(audit_log).not_to be_nil
+        expect(audit_log.user_id).to eq(user.id)
+        expect(audit_log.action).to eq("index")
+        expect(audit_log.resource_type).to eq("Report")
+        expect(audit_log.resource_id).to eq(nil)
+        expect(audit_log.resource).to eq(nil)
+        expect(audit_log.workspace_id).to eq(workspace.id)
+        expect(audit_log.created_at).not_to be_nil
+        expect(audit_log.updated_at).not_to be_nil
       end
 
       it "returns success and time slices for viewer role" do
@@ -98,6 +109,17 @@
         expect(total_sync_run_rows[slice_size - 1]["total_count"]).to eq(6)
         expect(total_sync_run_rows[slice_size - 1]["success_count"]).to eq(3)
         expect(total_sync_run_rows[slice_size - 1]["failed_count"]).to eq(3)
+
+        audit_log = AuditLog.last
+        expect(audit_log).not_to be_nil
+        expect(audit_log.user_id).to eq(user.id)
+        expect(audit_log.action).to eq("index")
+        expect(audit_log.resource_type).to eq("Report")
+        expect(audit_log.resource_id).to eq(nil)
+        expect(audit_log.resource).to eq(nil)
+        expect(audit_log.workspace_id).to eq(workspace.id)
+        expect(audit_log.created_at).not_to be_nil
+        expect(audit_log.updated_at).not_to be_nil
       end
 
       it "returns success and time slices for member role" do
@@ -123,6 +145,17 @@
         expect(total_sync_run_rows[slice_size - 1]["total_count"]).to eq(6)
         expect(total_sync_run_rows[slice_size - 1]["success_count"]).to eq(3)
         expect(total_sync_run_rows[slice_size - 1]["failed_count"]).to eq(3)
+
+        audit_log = AuditLog.last
+        expect(audit_log).not_to be_nil
+        expect(audit_log.user_id).to eq(user.id)
+        expect(audit_log.action).to eq("index")
+        expect(audit_log.resource_type).to eq("Report")
+        expect(audit_log.resource_id).to eq(nil)
+        expect(audit_log.resource).to eq(nil)
+        expect(audit_log.workspace_id).to eq(workspace.id)
+        expect(audit_log.created_at).not_to be_nil
+        expect(audit_log.updated_at).not_to be_nil
       end
     end
   end

From 00aab206f44280962910c3d5a85f587fd66d4b52 Mon Sep 17 00:00:00 2001
From: "github-actions[bot]"
 <41898282+github-actions[bot]@users.noreply.github.com>
Date: Tue, 29 Oct 2024 09:46:06 +0530
Subject: [PATCH 16/48] chore(CE): data app report model changes (#445)

Co-authored-by: afthab vp <afthab@multiwoven.com>
---
 server/app/models/data_app.rb       |  1 +
 server/app/models/feedback.rb       |  1 +
 server/spec/factories/feedback.rb   | 13 +++++++++++++
 server/spec/models/data_app_spec.rb |  1 +
 server/spec/models/feedback_spec.rb |  1 +
 5 files changed, 17 insertions(+)
 create mode 100644 server/spec/factories/feedback.rb

diff --git a/server/app/models/data_app.rb b/server/app/models/data_app.rb
index b6e42637..bd9c32a8 100644
--- a/server/app/models/data_app.rb
+++ b/server/app/models/data_app.rb
@@ -12,6 +12,7 @@ class DataApp < ApplicationRecord
   belongs_to :workspace
   has_many :visual_components, dependent: :destroy
   has_many :models, through: :visual_components
+  has_many :feedbacks, through: :visual_components
   has_many :data_app_sessions, dependent: :nullify
 
   after_initialize :set_default_status, if: :new_record?
diff --git a/server/app/models/feedback.rb b/server/app/models/feedback.rb
index 5493a173..8c311446 100644
--- a/server/app/models/feedback.rb
+++ b/server/app/models/feedback.rb
@@ -9,6 +9,7 @@ class Feedback < ApplicationRecord
   belongs_to :data_app
   belongs_to :visual_component
   belongs_to :model
+  belongs_to :workspace
 
   enum feedback_type: { thumbs: 0, scale_input: 1, text_input: 2, dropdown: 3 }
 
diff --git a/server/spec/factories/feedback.rb b/server/spec/factories/feedback.rb
new file mode 100644
index 00000000..907e16cd
--- /dev/null
+++ b/server/spec/factories/feedback.rb
@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+FactoryBot.define do
+  factory :feedback do
+    association :data_app
+    association :visual_component
+    association :workspace
+    association :model
+    session_id { SecureRandom.hex(10) }
+    feedback_type { "thumbs" }
+    reaction { "positive" }
+  end
+end
diff --git a/server/spec/models/data_app_spec.rb b/server/spec/models/data_app_spec.rb
index c863eadd..6cf1afdd 100644
--- a/server/spec/models/data_app_spec.rb
+++ b/server/spec/models/data_app_spec.rb
@@ -15,6 +15,7 @@
     it { should have_many(:visual_components).dependent(:destroy) }
     it { should have_many(:models).through(:visual_components) }
     it { should have_many(:data_app_sessions).dependent(:nullify) }
+    it { should have_many(:feedbacks).through(:visual_components) }
   end
 
   describe "#set_default_status" do
diff --git a/server/spec/models/feedback_spec.rb b/server/spec/models/feedback_spec.rb
index bf601d1f..b4361ea7 100644
--- a/server/spec/models/feedback_spec.rb
+++ b/server/spec/models/feedback_spec.rb
@@ -7,6 +7,7 @@
     it { should belong_to(:data_app) }
     it { should belong_to(:visual_component) }
     it { should belong_to(:model) }
+    it { should belong_to(:workspace) }
   end
 
   describe "validations" do

From 73a1987df7ae273241caf1da3d7c58c589457272 Mon Sep 17 00:00:00 2001
From: "github-actions[bot]"
 <41898282+github-actions[bot]@users.noreply.github.com>
Date: Wed, 30 Oct 2024 10:19:32 +0530
Subject: [PATCH 17/48] chore(CE): add audit logs to users controller (#449)

Co-authored-by: TivonB-AI2 <124182151+TivonB-AI2@users.noreply.github.com>
---
 .../controllers/api/v1/users_controller.rb    |  9 +++++
 .../requests/api/v1/users_controller_spec.rb  | 33 +++++++++++++++++++
 2 files changed, 42 insertions(+)

diff --git a/server/app/controllers/api/v1/users_controller.rb b/server/app/controllers/api/v1/users_controller.rb
index 84f27842..80925b18 100644
--- a/server/app/controllers/api/v1/users_controller.rb
+++ b/server/app/controllers/api/v1/users_controller.rb
@@ -3,10 +3,19 @@
 module Api
   module V1
     class UsersController < ApplicationController
+      include AuditLogger
+      after_action :create_audit_log
+
       def me
         authorize current_user
         render json: current_user, serializer: UserSerializer, workspace_id: current_workspace.id
       end
+
+      private
+
+      def create_audit_log
+        audit!(resource_id: params[:id])
+      end
     end
   end
 end
diff --git a/server/spec/requests/api/v1/users_controller_spec.rb b/server/spec/requests/api/v1/users_controller_spec.rb
index 51ec3637..5a750489 100644
--- a/server/spec/requests/api/v1/users_controller_spec.rb
+++ b/server/spec/requests/api/v1/users_controller_spec.rb
@@ -31,6 +31,17 @@
         expect(response_hash.dig(:data, :attributes, :created_at)).not_to be_nil
         expect(response_hash.dig(:data, :attributes, :role)).to eq("Admin")
         expect(response_hash.dig(:data, :attributes, :status)).to eq("active")
+
+        audit_log = AuditLog.last
+        expect(audit_log).not_to be_nil
+        expect(audit_log.user_id).to eq(user.id)
+        expect(audit_log.action).to eq("me")
+        expect(audit_log.resource_type).to eq("User")
+        expect(audit_log.resource_id).to eq(nil)
+        expect(audit_log.resource).to eq(nil)
+        expect(audit_log.workspace_id).to eq(workspace.id)
+        expect(audit_log.created_at).not_to be_nil
+        expect(audit_log.updated_at).not_to be_nil
       end
 
       it "returns user details withg member role" do
@@ -45,6 +56,17 @@
         expect(response_hash.dig(:data, :attributes, :created_at)).not_to be_nil
         expect(response_hash.dig(:data, :attributes, :role)).to eq("Member")
         expect(response_hash.dig(:data, :attributes, :status)).to eq("active")
+
+        audit_log = AuditLog.last
+        expect(audit_log).not_to be_nil
+        expect(audit_log.user_id).to eq(user.id)
+        expect(audit_log.action).to eq("me")
+        expect(audit_log.resource_type).to eq("User")
+        expect(audit_log.resource_id).to eq(nil)
+        expect(audit_log.resource).to eq(nil)
+        expect(audit_log.workspace_id).to eq(workspace.id)
+        expect(audit_log.created_at).not_to be_nil
+        expect(audit_log.updated_at).not_to be_nil
       end
 
       it "returns user details withg viewer role" do
@@ -59,6 +81,17 @@
         expect(response_hash.dig(:data, :attributes, :created_at)).not_to be_nil
         expect(response_hash.dig(:data, :attributes, :role)).to eq("Viewer")
         expect(response_hash.dig(:data, :attributes, :status)).to eq("active")
+
+        audit_log = AuditLog.last
+        expect(audit_log).not_to be_nil
+        expect(audit_log.user_id).to eq(user.id)
+        expect(audit_log.action).to eq("me")
+        expect(audit_log.resource_type).to eq("User")
+        expect(audit_log.resource_id).to eq(nil)
+        expect(audit_log.resource).to eq(nil)
+        expect(audit_log.workspace_id).to eq(workspace.id)
+        expect(audit_log.created_at).not_to be_nil
+        expect(audit_log.updated_at).not_to be_nil
       end
     end
   end

From 3723e23bf40a0fd8deff60f72184a739967cb7be Mon Sep 17 00:00:00 2001
From: "github-actions[bot]"
 <41898282+github-actions[bot]@users.noreply.github.com>
Date: Wed, 30 Oct 2024 10:20:06 +0530
Subject: [PATCH 18/48] chore(CE): add audit to sync records controller (#448)

Co-authored-by: TivonB-AI2 <124182151+TivonB-AI2@users.noreply.github.com>
---
 .../api/v1/sync_records_controller.rb         |  6 ++++
 .../api/v1/sync_records_controller_spec.rb    | 33 +++++++++++++++++++
 2 files changed, 39 insertions(+)

diff --git a/server/app/controllers/api/v1/sync_records_controller.rb b/server/app/controllers/api/v1/sync_records_controller.rb
index be7121f2..2d5b9bc2 100644
--- a/server/app/controllers/api/v1/sync_records_controller.rb
+++ b/server/app/controllers/api/v1/sync_records_controller.rb
@@ -3,8 +3,10 @@
 module Api
   module V1
     class SyncRecordsController < ApplicationController
+      include AuditLogger
       before_action :set_sync
       before_action :set_sync_run
+      after_action :create_audit_log
       attr_reader :sync, :sync_run
 
       def index
@@ -26,6 +28,10 @@ def set_sync_run
         @sync_run = @sync&.sync_runs&.find_by(id: params[:sync_run_id])
         render_error(message: "SyncRun not found", status: :not_found) unless @sync_run
       end
+
+      def create_audit_log
+        audit!
+      end
     end
   end
 end
diff --git a/server/spec/requests/api/v1/sync_records_controller_spec.rb b/server/spec/requests/api/v1/sync_records_controller_spec.rb
index 6e2626f0..00d8c7de 100644
--- a/server/spec/requests/api/v1/sync_records_controller_spec.rb
+++ b/server/spec/requests/api/v1/sync_records_controller_spec.rb
@@ -63,6 +63,17 @@
           end
           expect(response_hash.dig(:links, :first)).to include("http://www.example.com/api/v1/syncs/#{sync.id}/sync_runs/#{sync_run.id}/sync_records?page=1")
         end
+
+        audit_log = AuditLog.last
+        expect(audit_log).not_to be_nil
+        expect(audit_log.user_id).to eq(user.id)
+        expect(audit_log.action).to eq("index")
+        expect(audit_log.resource_type).to eq("Sync_record")
+        expect(audit_log.resource_id).to eq(nil)
+        expect(audit_log.resource).to eq(nil)
+        expect(audit_log.workspace_id).to eq(workspace.id)
+        expect(audit_log.created_at).not_to be_nil
+        expect(audit_log.updated_at).not_to be_nil
       end
 
       it "returns success and fetch sync for member role" do
@@ -88,6 +99,17 @@
           end
           expect(response_hash.dig(:links, :first)).to include("http://www.example.com/api/v1/syncs/#{sync.id}/sync_runs/#{sync_run.id}/sync_records?page=1")
         end
+
+        audit_log = AuditLog.last
+        expect(audit_log).not_to be_nil
+        expect(audit_log.user_id).to eq(user.id)
+        expect(audit_log.action).to eq("index")
+        expect(audit_log.resource_type).to eq("Sync_record")
+        expect(audit_log.resource_id).to eq(nil)
+        expect(audit_log.resource).to eq(nil)
+        expect(audit_log.workspace_id).to eq(workspace.id)
+        expect(audit_log.created_at).not_to be_nil
+        expect(audit_log.updated_at).not_to be_nil
       end
 
       it "returns success and fetch sync for viewer role" do
@@ -113,6 +135,17 @@
           end
           expect(response_hash.dig(:links, :first)).to include("http://www.example.com/api/v1/syncs/#{sync.id}/sync_runs/#{sync_run.id}/sync_records?page=1")
         end
+
+        audit_log = AuditLog.last
+        expect(audit_log).not_to be_nil
+        expect(audit_log.user_id).to eq(user.id)
+        expect(audit_log.action).to eq("index")
+        expect(audit_log.resource_type).to eq("Sync_record")
+        expect(audit_log.resource_id).to eq(nil)
+        expect(audit_log.resource).to eq(nil)
+        expect(audit_log.workspace_id).to eq(workspace.id)
+        expect(audit_log.created_at).not_to be_nil
+        expect(audit_log.updated_at).not_to be_nil
       end
 
       context "when it is an authenticated user and passing invalid parameters" do

From 0395e5c3be92373ffca38b1e95137b7d29d9a9f6 Mon Sep 17 00:00:00 2001
From: "github-actions[bot]"
 <41898282+github-actions[bot]@users.noreply.github.com>
Date: Wed, 30 Oct 2024 10:20:46 +0530
Subject: [PATCH 19/48] chore(CE): add audit logs to sync runs controller
 (#450)

Co-authored-by: TivonB-AI2 <124182151+TivonB-AI2@users.noreply.github.com>
---
 .../api/v1/sync_runs_controller.rb            |  6 ++
 .../api/v1/sync_runs_controller_spec.rb       | 66 +++++++++++++++++++
 2 files changed, 72 insertions(+)

diff --git a/server/app/controllers/api/v1/sync_runs_controller.rb b/server/app/controllers/api/v1/sync_runs_controller.rb
index b35a3374..91d8e7e3 100644
--- a/server/app/controllers/api/v1/sync_runs_controller.rb
+++ b/server/app/controllers/api/v1/sync_runs_controller.rb
@@ -3,8 +3,10 @@
 module Api
   module V1
     class SyncRunsController < ApplicationController
+      include AuditLogger
       before_action :set_sync
       before_action :set_sync_run, only: [:show]
+      after_action :create_audit_log
       attr_reader :sync
 
       def index
@@ -31,6 +33,10 @@ def set_sync_run
         @sync_run = @sync.sync_runs.find_by(id: params[:id])
         render_error(message: "Sync Run not found", status: :not_found) unless @sync_run
       end
+
+      def create_audit_log
+        audit!(resource_id: params[:id])
+      end
     end
   end
 end
diff --git a/server/spec/requests/api/v1/sync_runs_controller_spec.rb b/server/spec/requests/api/v1/sync_runs_controller_spec.rb
index 18454f4a..1dec1edd 100644
--- a/server/spec/requests/api/v1/sync_runs_controller_spec.rb
+++ b/server/spec/requests/api/v1/sync_runs_controller_spec.rb
@@ -53,6 +53,17 @@
           expect(row.dig(:attributes, :status)).to eq(sync_run.status)
           expect(response_hash.dig(:links, :first)).to include("http://www.example.com/api/v1/syncs/#{sync.id}/sync_runs?page=1")
         end
+
+        audit_log = AuditLog.last
+        expect(audit_log).not_to be_nil
+        expect(audit_log.user_id).to eq(user.id)
+        expect(audit_log.action).to eq("index")
+        expect(audit_log.resource_type).to eq("Sync_run")
+        expect(audit_log.resource_id).to eq(nil)
+        expect(audit_log.resource).to eq(nil)
+        expect(audit_log.workspace_id).to eq(workspace.id)
+        expect(audit_log.created_at).not_to be_nil
+        expect(audit_log.updated_at).not_to be_nil
       end
 
       it "returns success and fetch sync for member_role" do
@@ -72,6 +83,17 @@
           expect(row.dig(:attributes, :status)).to eq(sync_run.status)
           expect(response_hash.dig(:links, :first)).to include("http://www.example.com/api/v1/syncs/#{sync.id}/sync_runs?page=1")
         end
+
+        audit_log = AuditLog.last
+        expect(audit_log).not_to be_nil
+        expect(audit_log.user_id).to eq(user.id)
+        expect(audit_log.action).to eq("index")
+        expect(audit_log.resource_type).to eq("Sync_run")
+        expect(audit_log.resource_id).to eq(nil)
+        expect(audit_log.resource).to eq(nil)
+        expect(audit_log.workspace_id).to eq(workspace.id)
+        expect(audit_log.created_at).not_to be_nil
+        expect(audit_log.updated_at).not_to be_nil
       end
 
       it "returns success and fetch sync for viewer_role" do
@@ -91,6 +113,17 @@
           expect(row.dig(:attributes, :status)).to eq(sync_run.status)
           expect(response_hash.dig(:links, :first)).to include("http://www.example.com/api/v1/syncs/#{sync.id}/sync_runs?page=1")
         end
+
+        audit_log = AuditLog.last
+        expect(audit_log).not_to be_nil
+        expect(audit_log.user_id).to eq(user.id)
+        expect(audit_log.action).to eq("index")
+        expect(audit_log.resource_type).to eq("Sync_run")
+        expect(audit_log.resource_id).to eq(nil)
+        expect(audit_log.resource).to eq(nil)
+        expect(audit_log.workspace_id).to eq(workspace.id)
+        expect(audit_log.created_at).not_to be_nil
+        expect(audit_log.updated_at).not_to be_nil
       end
     end
 
@@ -139,6 +172,17 @@
         expect(response_hash.dig(:data, :attributes, :successful_rows)).to eq(sync_runs.first.successful_rows)
         expect(response_hash.dig(:data, :attributes, :failed_rows)).to eq(sync_runs.first.failed_rows)
         expect(response_hash.dig(:data, :attributes, :status)).to eq(sync_runs.first.status)
+
+        audit_log = AuditLog.last
+        expect(audit_log).not_to be_nil
+        expect(audit_log.user_id).to eq(user.id)
+        expect(audit_log.action).to eq("show")
+        expect(audit_log.resource_type).to eq("Sync_run")
+        expect(audit_log.resource_id).to eq(sync_runs.first.id)
+        expect(audit_log.resource).to eq(nil)
+        expect(audit_log.workspace_id).to eq(workspace.id)
+        expect(audit_log.created_at).not_to be_nil
+        expect(audit_log.updated_at).not_to be_nil
       end
 
       it "returns success and fetch sync for " do
@@ -153,6 +197,17 @@
         expect(response_hash.dig(:data, :attributes, :successful_rows)).to eq(sync_runs.first.successful_rows)
         expect(response_hash.dig(:data, :attributes, :failed_rows)).to eq(sync_runs.first.failed_rows)
         expect(response_hash.dig(:data, :attributes, :status)).to eq(sync_runs.first.status)
+
+        audit_log = AuditLog.last
+        expect(audit_log).not_to be_nil
+        expect(audit_log.user_id).to eq(user.id)
+        expect(audit_log.action).to eq("show")
+        expect(audit_log.resource_type).to eq("Sync_run")
+        expect(audit_log.resource_id).to eq(sync_runs.first.id)
+        expect(audit_log.resource).to eq(nil)
+        expect(audit_log.workspace_id).to eq(workspace.id)
+        expect(audit_log.created_at).not_to be_nil
+        expect(audit_log.updated_at).not_to be_nil
       end
 
       it "returns success and fetch sync " do
@@ -166,6 +221,17 @@
         expect(response_hash.dig(:data, :attributes, :successful_rows)).to eq(sync_runs.first.successful_rows)
         expect(response_hash.dig(:data, :attributes, :failed_rows)).to eq(sync_runs.first.failed_rows)
         expect(response_hash.dig(:data, :attributes, :status)).to eq(sync_runs.first.status)
+
+        audit_log = AuditLog.last
+        expect(audit_log).not_to be_nil
+        expect(audit_log.user_id).to eq(user.id)
+        expect(audit_log.action).to eq("show")
+        expect(audit_log.resource_type).to eq("Sync_run")
+        expect(audit_log.resource_id).to eq(sync_runs.first.id)
+        expect(audit_log.resource).to eq(nil)
+        expect(audit_log.workspace_id).to eq(workspace.id)
+        expect(audit_log.created_at).not_to be_nil
+        expect(audit_log.updated_at).not_to be_nil
       end
     end
 

From fdb0d241090fa9636f60f02b57a70275116edcd3 Mon Sep 17 00:00:00 2001
From: "Rafael E. O'Neill"
 <106079170+RafaelOAiSquared@users.noreply.github.com>
Date: Wed, 30 Oct 2024 01:39:59 -0400
Subject: [PATCH 20/48] Multiwoven release v0.31.0 (#447)

Co-authored-by: github-actions <developer@multiwoven.com>
---
 release-notes.md | 27 +++++++++++++++++----------
 1 file changed, 17 insertions(+), 10 deletions(-)

diff --git a/release-notes.md b/release-notes.md
index 7ae8dc6d..dbc205e6 100644
--- a/release-notes.md
+++ b/release-notes.md
@@ -2,21 +2,28 @@
 
 All notable changes to this project will be documented in this file.
 
-## [0.30.0] - 2024-10-21
+## [0.31.0] - 2024-10-28
+
+### 🚀 Features
+
+- *(CE)* Added new pagination format
 
 ### 🐛 Bug Fixes
 
-- *(CE)* Server ci set ubuntu to 22.04 (#411)
-- *(CE)* Disable the database middle ware for temporal (#413)
-- *(CE)* Dashboard sidebar responsiveness
-- *(CE)* Wrong argument type NoMethodError (expected String) (#415)
+- *(CE)* Change the documentation link
+- *(CE)* Add exception to tracker (#436)
 
 ### ⚙️ Miscellaneous Tasks
 
-- *(CE)* Configure user email verification based on env variable (#409)
-- *(CE)* Audit Model(DB) Changes (#416)
-- *(CE)* Feedback model changes (#414)
-- *(CE)* Reaction field enum correction (#491) (#418)
-- *(CE)* Add audit logger and audit to syncs controller (#417)
+- *(CE)* Add audit log into syncs controller (#422)
+- *(CE)* Add audit log to connectors controller (#424)
+- *(CE)* Add audit log to connector definitions controller (#421)
+- *(CE)* Add audit logger and audit to models controller (#425)
+- *(CE)* Add audit logs serializer and audit log permission (#426)
+- *(CE)* Added audit logs to catalogs_controller (#439)
+- *(CE)* Added audit logs to schedule_sync_controller (#440)
+- *(CE)* Add data app session model (#438)
+- *(CE)* Added audit log to workspace (#442)
+- *(CE)* Add audit logs to factory and roles (#441)
 
 <!-- generated by git-cliff -->

From 9a46d875ccee42934827676679a104ba58a8ac2b Mon Sep 17 00:00:00 2001
From: "Rafael E. O'Neill"
 <106079170+RafaelOAiSquared@users.noreply.github.com>
Date: Tue, 5 Nov 2024 12:18:45 -0400
Subject: [PATCH 21/48] Multiwoven release v0.32.0 (#453)

Co-authored-by: github-actions <developer@multiwoven.com>
---
 release-notes.md | 26 ++++++--------------------
 1 file changed, 6 insertions(+), 20 deletions(-)

diff --git a/release-notes.md b/release-notes.md
index dbc205e6..d9c649fc 100644
--- a/release-notes.md
+++ b/release-notes.md
@@ -2,28 +2,14 @@
 
 All notable changes to this project will be documented in this file.
 
-## [0.31.0] - 2024-10-28
-
-### 🚀 Features
-
-- *(CE)* Added new pagination format
-
-### 🐛 Bug Fixes
-
-- *(CE)* Change the documentation link
-- *(CE)* Add exception to tracker (#436)
+## [0.32.0] - 2024-10-30
 
 ### ⚙️ Miscellaneous Tasks
 
-- *(CE)* Add audit log into syncs controller (#422)
-- *(CE)* Add audit log to connectors controller (#424)
-- *(CE)* Add audit log to connector definitions controller (#421)
-- *(CE)* Add audit logger and audit to models controller (#425)
-- *(CE)* Add audit logs serializer and audit log permission (#426)
-- *(CE)* Added audit logs to catalogs_controller (#439)
-- *(CE)* Added audit logs to schedule_sync_controller (#440)
-- *(CE)* Add data app session model (#438)
-- *(CE)* Added audit log to workspace (#442)
-- *(CE)* Add audit logs to factory and roles (#441)
+- *(CE)* Add audit logs to report controller (#446)
+- *(CE)* Data app report model changes (#445)
+- *(CE)* Add audit logs to users controller (#449)
+- *(CE)* Add audit to sync records controller (#448)
+- *(CE)* Add audit logs to sync runs controller (#450)
 
 <!-- generated by git-cliff -->

From 924daa76f5565626fc22d3c67759c54ececbaba7 Mon Sep 17 00:00:00 2001
From: "github-actions[bot]"
 <41898282+github-actions[bot]@users.noreply.github.com>
Date: Fri, 8 Nov 2024 10:45:54 +0530
Subject: [PATCH 22/48] chore(CE): add additional_remarks in feedback model
 (#454)

Co-authored-by: afthab vp <afthab@multiwoven.com>
---
 .../20241107074907_add_additional_remarks_to_feedbacks.rb    | 5 +++++
 server/db/schema.rb                                          | 3 ++-
 2 files changed, 7 insertions(+), 1 deletion(-)
 create mode 100644 server/db/migrate/20241107074907_add_additional_remarks_to_feedbacks.rb

diff --git a/server/db/migrate/20241107074907_add_additional_remarks_to_feedbacks.rb b/server/db/migrate/20241107074907_add_additional_remarks_to_feedbacks.rb
new file mode 100644
index 00000000..16b47d2a
--- /dev/null
+++ b/server/db/migrate/20241107074907_add_additional_remarks_to_feedbacks.rb
@@ -0,0 +1,5 @@
+class AddAdditionalRemarksToFeedbacks < ActiveRecord::Migration[7.1]
+  def change
+    add_column :feedbacks, :additional_remarks, :jsonb
+  end
+end
diff --git a/server/db/schema.rb b/server/db/schema.rb
index 7de8dcfc..a1cb113f 100644
--- a/server/db/schema.rb
+++ b/server/db/schema.rb
@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema[7.1].define(version: 2024_10_24_103527) do
+ActiveRecord::Schema[7.1].define(version: 2024_11_07_074907) do
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
 
@@ -86,6 +86,7 @@
     t.datetime "updated_at", null: false
     t.integer "feedback_type", default: 0, null: false
     t.string "session_id"
+    t.jsonb "additional_remarks"
   end
 
   create_table "models", force: :cascade do |t|

From b0a18be380a2bb6f47b667d50ea10c4b4ff79668 Mon Sep 17 00:00:00 2001
From: Naveen  yadav <52673659+xyfer17@users.noreply.github.com>
Date: Mon, 11 Nov 2024 10:45:01 +0530
Subject: [PATCH 23/48] feat(CE): Added Mailchimp Destination Connector (#437)

* feat(CE): Added Mailchimp Destination Connector
---
 integrations/Gemfile                          |   2 +
 integrations/Gemfile.lock                     |   5 +
 integrations/lib/multiwoven/integrations.rb   |   2 +
 .../destination/mailchimp/client.rb           | 141 +++++++++++++
 .../destination/mailchimp/config/catalog.json | 141 +++++++++++++
 .../destination/mailchimp/config/meta.json    |  15 ++
 .../destination/mailchimp/config/spec.json    |  28 +++
 .../destination/mailchimp/icon.svg            |   4 +
 .../lib/multiwoven/integrations/rollout.rb    |   3 +-
 integrations/multiwoven-integrations.gemspec  |   1 +
 .../destination/mailchimp/client_spec.rb      | 185 ++++++++++++++++++
 11 files changed, 526 insertions(+), 1 deletion(-)
 create mode 100644 integrations/lib/multiwoven/integrations/destination/mailchimp/client.rb
 create mode 100644 integrations/lib/multiwoven/integrations/destination/mailchimp/config/catalog.json
 create mode 100644 integrations/lib/multiwoven/integrations/destination/mailchimp/config/meta.json
 create mode 100644 integrations/lib/multiwoven/integrations/destination/mailchimp/config/spec.json
 create mode 100644 integrations/lib/multiwoven/integrations/destination/mailchimp/icon.svg
 create mode 100644 integrations/spec/multiwoven/integrations/destination/mailchimp/client_spec.rb

diff --git a/integrations/Gemfile b/integrations/Gemfile
index 74553c43..45940f9a 100644
--- a/integrations/Gemfile
+++ b/integrations/Gemfile
@@ -77,6 +77,8 @@ gem "google-cloud-ai_platform-v1"
 
 gem "tiny_tds"
 
+gem "MailchimpMarketing"
+
 group :development, :test do
   gem "simplecov", require: false
   gem "simplecov_json_formatter", require: false
diff --git a/integrations/Gemfile.lock b/integrations/Gemfile.lock
index 3861d551..a510a347 100644
--- a/integrations/Gemfile.lock
+++ b/integrations/Gemfile.lock
@@ -44,6 +44,9 @@ PATH
 GEM
   remote: https://rubygems.org/
   specs:
+    MailchimpMarketing (3.0.80)
+      excon (>= 0.76.0, < 1)
+      json (~> 2.1, >= 2.1.0)
     activesupport (7.1.3.3)
       base64
       bigdecimal
@@ -151,6 +154,7 @@ GEM
       bigdecimal (>= 3.1.4)
     ethon (0.16.0)
       ffi (>= 1.15.0)
+    excon (0.112.0)
     faraday (2.8.1)
       base64
       faraday-net_http (>= 2.0, < 3.1)
@@ -407,6 +411,7 @@ PLATFORMS
   x86_64-linux
 
 DEPENDENCIES
+  MailchimpMarketing
   activesupport
   async-websocket (~> 0.8.0)
   aws-sdk-athena
diff --git a/integrations/lib/multiwoven/integrations.rb b/integrations/lib/multiwoven/integrations.rb
index 32a1bf58..d64b7d96 100644
--- a/integrations/lib/multiwoven/integrations.rb
+++ b/integrations/lib/multiwoven/integrations.rb
@@ -36,6 +36,7 @@
 require "aws-sdk-sagemakerruntime"
 require "google/cloud/ai_platform/v1"
 require "grpc"
+require "MailchimpMarketing"
 
 # Service
 require_relative "integrations/config"
@@ -90,6 +91,7 @@
 require_relative "integrations/destination/oracle_db/client"
 require_relative "integrations/destination/microsoft_excel/client"
 require_relative "integrations/destination/microsoft_sql/client"
+require_relative "integrations/destination/mailchimp/client"
 
 module Multiwoven
   module Integrations
diff --git a/integrations/lib/multiwoven/integrations/destination/mailchimp/client.rb b/integrations/lib/multiwoven/integrations/destination/mailchimp/client.rb
new file mode 100644
index 00000000..cfb56f01
--- /dev/null
+++ b/integrations/lib/multiwoven/integrations/destination/mailchimp/client.rb
@@ -0,0 +1,141 @@
+# frozen_string_literal: true
+
+module Multiwoven
+  module Integrations
+    module Destination
+      module Mailchimp
+        include Multiwoven::Integrations::Core
+
+        API_VERSION = "3.0"
+
+        class Client < DestinationConnector
+          prepend Multiwoven::Integrations::Core::RateLimiter
+
+          def check_connection(connection_config)
+            connection_config = connection_config.with_indifferent_access
+            initialize_client(connection_config)
+            authenticate_client
+            success_status
+          rescue StandardError => e
+            failure_status(e)
+          end
+
+          def discover(_connection_config = nil)
+            catalog = build_catalog(load_catalog)
+            catalog.to_multiwoven_message
+          rescue StandardError => e
+            handle_exception(e, {
+                               context: "MAILCHIMP:DISCOVER:EXCEPTION",
+                               type: "error"
+                             })
+          end
+
+          def write(sync_config, records, _action = "create")
+            @sync_config = sync_config
+            initialize_client(sync_config.destination.connection_specification)
+            process_records(records, sync_config.stream)
+          rescue StandardError => e
+            handle_exception(e, {
+                               context: "MAILCHIMP:WRITE:EXCEPTION",
+                               type: "error",
+                               sync_id: @sync_config.sync_id,
+                               sync_run_id: @sync_config.sync_run_id
+                             })
+          end
+
+          private
+
+          def initialize_client(config)
+            config = config.with_indifferent_access
+            @client = MailchimpMarketing::Client.new
+            @client.set_config({
+                                 api_key: config[:api_key],
+                                 server: config[:api_key].split("-").last
+                               })
+            @list_id = config[:list_id]
+            @email_template_id = config[:email_template_id] || ""
+          end
+
+          def process_records(records, stream)
+            log_message_array = []
+            write_success = 0
+            write_failure = 0
+            properties = stream.json_schema[:properties]
+
+            records.each do |record_object|
+              record = extract_data(record_object, properties)
+              args = [stream.name, "Id", record]
+              begin
+                response = send_to_mailchimp(record, stream.name)
+                write_success += 1
+                log_message_array << log_request_response("info", args, response)
+              rescue StandardError => e
+                handle_exception(e, {
+                                   context: "MAILCHIMP:WRITE:EXCEPTION",
+                                   type: "error",
+                                   sync_id: @sync_config.sync_id,
+                                   sync_run_id: @sync_config.sync_run_id
+                                 })
+                write_failure += 1
+                log_message_array << log_request_response("error", args, e.message)
+              end
+            end
+            tracking_message(write_success, write_failure, log_message_array)
+          end
+
+          def send_to_mailchimp(record, stream_name)
+            case stream_name
+            when "Audience"
+              @client.lists.set_list_member(@list_id, Digest::MD5.hexdigest(record[:email].downcase), {
+                                              email_address: record[:email],
+                                              status_if_new: "subscribed",
+                                              merge_fields: {
+                                                FNAME: record[:first_name],
+                                                LNAME: record[:last_name]
+                                              }
+                                            })
+            when "Tags"
+              @client.lists.update_list_member_tags(@list_id, Digest::MD5.hexdigest(record[:email].downcase), {
+                                                      tags: record[:tags].map { |tag| { name: tag, status: "active" } }
+                                                    })
+            when "Campaigns"
+              campaign = @client.campaigns.create({
+                                                    type: "regular",
+                                                    recipients: { list_id: @list_id },
+                                                    settings: {
+                                                      subject_line: record[:subject],
+                                                      from_name: record[:from_name],
+                                                      reply_to: record[:reply_to]
+                                                    }
+                                                  })
+              if @email_template_id
+                @client.campaigns.set_content(campaign["id"], {
+                                                template: { id: @email_template_id }
+                                              })
+              else
+                @client.campaigns.set_content(campaign["id"], {
+                                                plain_text: record[:content]
+                                              })
+              end
+              @client.campaigns.send(campaign["id"])
+            else
+              raise "Unsupported stream type: #{stream_name}"
+            end
+          end
+
+          def authenticate_client
+            @client.lists.get_all_lists
+          end
+
+          def load_catalog
+            read_json(CATALOG_SPEC_PATH)
+          end
+
+          def log_debug(message)
+            Multiwoven::Integrations::Service.logger.debug(message)
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/integrations/lib/multiwoven/integrations/destination/mailchimp/config/catalog.json b/integrations/lib/multiwoven/integrations/destination/mailchimp/config/catalog.json
new file mode 100644
index 00000000..58f86ccd
--- /dev/null
+++ b/integrations/lib/multiwoven/integrations/destination/mailchimp/config/catalog.json
@@ -0,0 +1,141 @@
+{
+  "request_rate_limit": 100000,
+  "request_rate_limit_unit": "day",
+  "request_rate_concurrency": 10,
+  "streams": [
+    {
+      "name": "Audience",
+      "action": "create",
+      "json_schema": {
+        "type": "object",
+        "additionalProperties": true,
+        "required": ["email"],
+        "properties": {
+          "email": {
+            "type": "string",
+            "format": "email"
+          },
+          "first_name": {
+            "type": "string"
+          },
+          "last_name": {
+            "type": "string"
+          },
+          "status": {
+            "type": "string",
+            "enum": ["subscribed", "unsubscribed", "cleaned", "pending"]
+          },
+          "tags": {
+            "type": "array",
+            "items": {
+              "type": "string"
+            }
+          },
+          "merge_fields": {
+            "type": "object",
+            "additionalProperties": true,
+            "properties": {
+              "FNAME": {
+                "type": "string"
+              },
+              "LNAME": {
+                "type": "string"
+              }
+            }
+          },
+          "language": {
+            "type": "string"
+          },
+          "vip": {
+            "type": "boolean"
+          },
+          "timestamp_signup": {
+            "type": "string",
+            "format": "date-time"
+          },
+          "ip_signup": {
+            "type": "string",
+            "format": "ipv4"
+          },
+          "timestamp_opt": {
+            "type": "string",
+            "format": "date-time"
+          },
+          "ip_opt": {
+            "type": "string",
+            "format": "ipv4"
+          }
+        }
+      },
+      "supported_sync_modes": ["incremental"],
+      "source_defined_cursor": true,
+      "default_cursor_field": ["timestamp_opt"],
+      "source_defined_primary_key": [["email"]]
+    },
+    {
+      "name": "Tags",
+      "action": "create",
+      "json_schema": {
+        "type": "object",
+        "additionalProperties": true,
+        "required": ["email", "tags"],
+        "properties": {
+          "email": {
+            "type": "string",
+            "format": "email"
+          },
+          "tags": {
+            "type": "array",
+            "items": {
+              "type": "string"
+            }
+          }
+        }
+      },
+      "supported_sync_modes": ["incremental"],
+      "source_defined_cursor": true,
+      "default_cursor_field": ["updated"]
+    },
+    {
+      "name": "Campaigns",
+      "action": "create",
+      "json_schema": {
+        "type": "object",
+        "additionalProperties": true,
+        "required": ["subject", "from_name", "reply_to", "recipients"],
+        "properties": {
+          "subject": {
+            "type": "string"
+          },
+          "from_name": {
+            "type": "string"
+          },
+          "reply_to": {
+            "type": "string",
+            "format": "email"
+          },
+          "recipients": {
+            "type": "object",
+            "properties": {
+              "list_id": {
+                "type": "string"
+              }
+            }
+          },
+          "template_id": {
+            "type": "string"
+          },
+          "content": {
+            "type": "string"
+          },
+          "send_time": {
+            "type": "string",
+            "format": "date-time"
+          }
+        }
+      },
+      "supported_sync_modes": ["full_refresh"],
+      "source_defined_cursor": false
+    }
+  ]
+}
diff --git a/integrations/lib/multiwoven/integrations/destination/mailchimp/config/meta.json b/integrations/lib/multiwoven/integrations/destination/mailchimp/config/meta.json
new file mode 100644
index 00000000..74e7f842
--- /dev/null
+++ b/integrations/lib/multiwoven/integrations/destination/mailchimp/config/meta.json
@@ -0,0 +1,15 @@
+{
+  "data": {
+    "name": "Mailchimp",
+    "title": "Mailchimp",
+    "connector_type": "destination",
+    "category": "Marketing Automation",
+    "documentation_url": "https://docs.multiwoven.com/destinations/crm/mailchimp",
+    "github_issue_label": "destination-mailchimp",
+    "icon": "icon.svg",
+    "license": "MIT",
+    "release_stage": "alpha",
+    "support_level": "community",
+    "tags": ["language:ruby", "multiwoven"]
+  }
+}
diff --git a/integrations/lib/multiwoven/integrations/destination/mailchimp/config/spec.json b/integrations/lib/multiwoven/integrations/destination/mailchimp/config/spec.json
new file mode 100644
index 00000000..1cc28e48
--- /dev/null
+++ b/integrations/lib/multiwoven/integrations/destination/mailchimp/config/spec.json
@@ -0,0 +1,28 @@
+{
+  "documentation_url": "https://docs.multiwoven.com/integrations/destination/mailchimp",
+  "stream_type": "static",
+  "connection_specification": {
+    "$schema": "http://json-schema.org/draft-07/schema#",
+    "title": "Mailchimp",
+    "type": "object",
+    "required": ["api_key", "list_id"],
+    "properties": {
+      "api_key": {
+        "type": "string",
+        "multiwoven_secret": true,
+        "title": "API Key",
+        "order": 0
+      },
+      "list_id": {
+        "type": "string",
+        "title": "List Id",
+        "order": 1
+      },
+      "email_template_id": {
+        "type": "string",
+        "title": "Email Template Id",
+        "order": 2
+      }
+    }
+  }
+}
diff --git a/integrations/lib/multiwoven/integrations/destination/mailchimp/icon.svg b/integrations/lib/multiwoven/integrations/destination/mailchimp/icon.svg
new file mode 100644
index 00000000..228a95e7
--- /dev/null
+++ b/integrations/lib/multiwoven/integrations/destination/mailchimp/icon.svg
@@ -0,0 +1,4 @@
+<?xml version="1.0" encoding="utf-8"?><!-- Uploaded to: SVG Repo, www.svgrepo.com, Generator: SVG Repo Mixer Tools -->
+<svg fill="#000000" width="800px" height="800px" viewBox="0 0 32 32" xmlns="http://www.w3.org/2000/svg">
+  <path d="M23.974 15.12c0.219-0.026 0.443-0.026 0.661 0 0.12-0.271 0.141-0.745 0.031-1.255-0.156-0.76-0.37-1.224-0.818-1.151-0.443 0.073-0.458 0.625-0.302 1.385 0.094 0.427 0.25 0.797 0.427 1.021zM20.161 15.724c0.318 0.141 0.51 0.234 0.589 0.151 0.047-0.052 0.036-0.146-0.042-0.271-0.156-0.255-0.479-0.516-0.823-0.661-0.719-0.302-1.547-0.203-2.177 0.26-0.214 0.156-0.411 0.375-0.385 0.505 0.010 0.042 0.042 0.073 0.115 0.083 0.177 0.021 0.786-0.286 1.495-0.333 0.495-0.031 0.911 0.125 1.229 0.266zM19.521 16.089c-0.417 0.068-0.641 0.203-0.786 0.328-0.13 0.109-0.203 0.234-0.203 0.318l0.031 0.078 0.068 0.026c0.094 0 0.302-0.089 0.302-0.089 0.427-0.156 0.885-0.208 1.333-0.135 0.208 0.021 0.307 0.036 0.354-0.036 0.010-0.021 0.026-0.063-0.016-0.13-0.094-0.161-0.516-0.427-1.083-0.359zM22.688 17.432c0.281 0.135 0.589 0.083 0.688-0.12 0.104-0.208-0.042-0.49-0.323-0.625-0.281-0.141-0.589-0.083-0.693 0.12-0.099 0.208 0.047 0.484 0.328 0.625zM24.495 15.849c-0.229-0.005-0.422 0.245-0.422 0.562-0.005 0.313 0.172 0.568 0.401 0.573s0.417-0.245 0.422-0.557c0.005-0.313-0.177-0.573-0.401-0.578zM9.156 21.505c-0.057-0.068-0.151-0.047-0.24-0.026-0.073 0.016-0.146 0.026-0.214 0.031-0.156 0-0.302-0.073-0.391-0.198-0.104-0.161-0.099-0.401 0.016-0.672l0.052-0.125c0.188-0.411 0.49-1.099 0.146-1.755-0.255-0.495-0.677-0.802-1.188-0.865-0.49-0.063-0.974 0.109-1.313 0.469-0.505 0.557-0.583 1.318-0.484 1.589 0.036 0.099 0.094 0.125 0.135 0.13 0.083 0.010 0.208-0.052 0.292-0.266l0.021-0.068c0.036-0.115 0.104-0.328 0.208-0.495 0.281-0.427 0.849-0.547 1.271-0.266 0.354 0.229 0.495 0.667 0.344 1.078-0.078 0.219-0.208 0.625-0.177 0.964 0.057 0.682 0.474 0.953 0.849 0.984 0.365 0.016 0.62-0.193 0.688-0.344 0.036-0.089 0.005-0.141-0.016-0.167zM30.255 20.26c-0.016-0.052-0.104-0.385-0.229-0.781l-0.255-0.682c0.5-0.75 0.51-1.422 0.443-1.802-0.073-0.469-0.266-0.87-0.661-1.281-0.391-0.417-1.198-0.844-2.333-1.161l-0.594-0.161c0-0.026-0.031-1.406-0.057-1.995-0.016-0.427-0.052-1.099-0.26-1.755-0.245-0.891-0.677-1.672-1.214-2.172 1.479-1.536 2.406-3.229 2.401-4.677-0.005-2.797-3.427-3.641-7.646-1.891l-0.896 0.38c-0.547-0.536-1.089-1.073-1.641-1.609-4.813-4.203-19.854 12.542-15.047 16.615l1.052 0.891c-0.281 0.76-0.38 1.578-0.292 2.391 0.115 1.12 0.688 2.193 1.625 3.021 0.885 0.786 2.052 1.286 3.188 1.286 1.87 4.318 6.146 6.969 11.156 7.12 5.38 0.156 9.896-2.37 11.786-6.911 0.125-0.318 0.651-1.755 0.651-3.021 0-1.276-0.719-1.802-1.177-1.802zM8.25 23.656c-0.161 0.031-0.333 0.042-0.5 0.036-1.62-0.042-3.375-1.505-3.552-3.245-0.193-1.922 0.786-3.396 2.521-3.745 0.24-0.052 0.484-0.068 0.729-0.057 0.974 0.057 2.406 0.802 2.729 2.922 0.292 1.88-0.167 3.792-1.927 4.094zM6.438 15.552c-1.078 0.208-2.031 0.823-2.615 1.667-0.349-0.286-0.995-0.849-1.109-1.068-0.927-1.771 1.016-5.203 2.37-7.146 3.354-4.792 8.609-8.422 11.042-7.766 0.396 0.115 1.703 1.635 1.703 1.635s-2.432 1.349-4.688 3.234c-3.036 2.344-5.328 5.745-6.703 9.443zM23.484 22.943c0.036-0.016 0.063-0.057 0.057-0.094-0.005-0.052-0.047-0.089-0.099-0.083 0 0-2.542 0.38-4.948-0.505 0.26-0.849 0.958-0.542 2.010-0.458 1.635 0.099 3.276-0.078 4.854-0.526 1.089-0.313 2.516-0.927 3.63-1.807 0.375 0.823 0.505 1.729 0.505 1.729s0.292-0.052 0.531 0.099c0.229 0.141 0.396 0.438 0.281 1.193-0.229 1.417-0.833 2.568-1.844 3.63-0.63 0.682-1.38 1.245-2.219 1.656-0.464 0.245-0.948 0.448-1.443 0.615-3.813 1.245-7.714-0.125-8.974-3.068-0.099-0.224-0.182-0.458-0.25-0.698-0.536-1.938-0.083-4.266 1.344-5.729 0.083-0.094 0.172-0.203 0.172-0.344 0-0.115-0.073-0.234-0.135-0.323-0.495-0.724-2.219-1.953-1.875-4.339 0.25-1.714 1.745-2.917 3.141-2.849l0.354 0.021c0.604 0.036 1.13 0.115 1.625 0.135 0.833 0.036 1.583-0.083 2.469-0.823 0.297-0.25 0.536-0.469 0.943-0.536 0.042-0.005 0.151-0.047 0.359-0.036 0.219 0.010 0.427 0.078 0.609 0.198 0.714 0.474 0.813 1.62 0.849 2.458 0.021 0.479 0.078 1.635 0.099 1.969 0.047 0.76 0.245 0.87 0.646 1 0.229 0.078 0.443 0.135 0.75 0.219 0.943 0.266 1.5 0.536 1.849 0.88 0.214 0.214 0.307 0.443 0.339 0.661 0.115 0.813-0.625 1.813-2.583 2.724-2.141 0.995-4.734 1.245-6.526 1.047l-0.63-0.073c-1.432-0.193-2.25 1.661-1.391 2.938 0.552 0.818 2.068 1.354 3.578 1.354 3.464 0 6.13-1.484 7.12-2.766l0.078-0.115c0.052-0.073 0.010-0.109-0.052-0.068-0.807 0.552-4.406 2.755-8.25 2.094 0 0-0.469-0.078-0.896-0.245-0.339-0.13-1.047-0.458-1.135-1.188 3.104 0.964 5.057 0.052 5.057 0.052zM18.568 22.359h0.005zM12.63 8.984c1.193-1.38 2.661-2.578 3.974-3.25 0.047-0.026 0.094 0.026 0.073 0.068-0.104 0.188-0.307 0.599-0.37 0.901-0.010 0.047 0.042 0.083 0.078 0.057 0.823-0.557 2.245-1.156 3.495-1.234 0.052 0 0.078 0.068 0.036 0.099-0.208 0.161-0.391 0.344-0.552 0.552-0.026 0.036 0 0.083 0.042 0.083 0.88 0.010 2.115 0.318 2.917 0.771 0.057 0.031 0.021 0.135-0.042 0.12-1.219-0.281-3.214-0.495-5.286 0.016-1.849 0.453-3.26 1.146-4.292 1.901-0.052 0.036-0.115-0.031-0.073-0.083z"/>
+</svg>
\ No newline at end of file
diff --git a/integrations/lib/multiwoven/integrations/rollout.rb b/integrations/lib/multiwoven/integrations/rollout.rb
index 26ab3073..b2f6dd7f 100644
--- a/integrations/lib/multiwoven/integrations/rollout.rb
+++ b/integrations/lib/multiwoven/integrations/rollout.rb
@@ -2,7 +2,7 @@
 
 module Multiwoven
   module Integrations
-    VERSION = "0.12.0"
+    VERSION = "0.13.0"
 
     ENABLED_SOURCES = %w[
       Snowflake
@@ -41,6 +41,7 @@ module Integrations
       Oracle
       MicrosoftExcel
       MicrosoftSql
+      Mailchimp
     ].freeze
   end
 end
diff --git a/integrations/multiwoven-integrations.gemspec b/integrations/multiwoven-integrations.gemspec
index ece63499..8a4174d1 100644
--- a/integrations/multiwoven-integrations.gemspec
+++ b/integrations/multiwoven-integrations.gemspec
@@ -67,6 +67,7 @@ Gem::Specification.new do |spec|
   spec.add_runtime_dependency "zendesk_api"
 
   spec.add_development_dependency "byebug"
+  spec.add_development_dependency "MailchimpMarketing"
   spec.add_development_dependency "rspec"
   spec.add_development_dependency "rubocop"
   spec.add_development_dependency "simplecov"
diff --git a/integrations/spec/multiwoven/integrations/destination/mailchimp/client_spec.rb b/integrations/spec/multiwoven/integrations/destination/mailchimp/client_spec.rb
new file mode 100644
index 00000000..2534025f
--- /dev/null
+++ b/integrations/spec/multiwoven/integrations/destination/mailchimp/client_spec.rb
@@ -0,0 +1,185 @@
+# frozen_string_literal: true
+
+RSpec.describe Multiwoven::Integrations::Destination::Mailchimp::Client do
+  include WebMock::API
+
+  before(:each) do
+    WebMock.disable_net_connect!(allow_localhost: true)
+  end
+
+  let(:client) { described_class.new }
+  let(:api_key) { "api_key" }
+  let(:list_id) { "list_id" }
+  let(:server) { api_key.split("-").last }
+  let(:email_template_id) { "email_template_id" }
+  let(:connection_config) do
+    {
+      api_key: api_key,
+      list_id: list_id,
+      email_template_id: email_template_id
+    }
+  end
+
+  let(:mailchimp_audience_json_schema) do
+    catalog = client.discover.catalog
+    catalog.streams.find { |stream| stream.name == "Audience" }.json_schema
+  end
+
+  let(:mailchimp_tags_json_schema) do
+    catalog = client.discover.catalog
+    catalog.streams.find { |stream| stream.name == "Tags" }.json_schema
+  end
+
+  let(:mailchimp_capaigns_json_schema) do
+    catalog = client.discover.catalog
+    catalog.streams.find { |stream| stream.name == "Campaigns" }.json_schema
+  end
+
+  let(:sync_config_json) do
+    { source: {
+        name: "SourceConnectorName",
+        type: "source",
+        connection_specification: {
+          private_api_key: "test_api_key"
+        }
+      },
+      destination: {
+        name: "Mailchimp",
+        type: "destination",
+        connection_specification: connection_config
+      },
+      model: {
+        name: "ExampleModel",
+        query: "SELECT * FROM CALL_CENTER LIMIT 1",
+        query_type: "raw_sql",
+        primary_key: "id"
+      },
+      stream: {
+        name: "Audience",
+        action: "create",
+        request_rate_limit: 4,
+        rate_limit_unit_seconds: 1,
+        json_schema: mailchimp_audience_json_schema
+      },
+      sync_mode: "incremental",
+      cursor_field: "timestamp",
+      destination_sync_mode: "insert" }.with_indifferent_access
+  end
+
+  let(:records) do
+    [
+      { email: "jane@example.com",
+        first_name: "Jane",
+        last_name: "Doe" },
+      { email: "jane@example.com",
+        first_name: "Jane",
+        last_name: "Doe" }
+    ]
+  end
+
+  describe "#check_connection" do
+    context "when the connection is successful" do
+      before do
+        stub_request(:get, "https://#{server}.api.mailchimp.com/3.0/lists")
+          .to_return(status: 200, body: "", headers: {})
+      end
+
+      it "returns a successful connection status" do
+        allow(client).to receive(:authenticate_client).and_return(true)
+
+        response = client.check_connection(connection_config)
+
+        expect(response).to be_a(Multiwoven::Integrations::Protocol::MultiwovenMessage)
+        expect(response.connection_status.status).to eq("succeeded")
+      end
+    end
+
+    context "when the connection fails" do
+      it "returns a failed connection status with an error message" do
+        allow(client).to receive(:authenticate_client).and_raise(StandardError.new("connection failed"))
+
+        response = client.check_connection(connection_config)
+
+        expect(response).to be_a(Multiwoven::Integrations::Protocol::MultiwovenMessage)
+        expect(response.connection_status.status).to eq("failed")
+        expect(response.connection_status.message).to eq("connection failed")
+      end
+    end
+  end
+
+  describe "#write" do
+    let(:list_member_url) { "https://#{server}.api.mailchimp.com/3.0/lists/#{list_id}/members" }
+
+    context "when the write operation is successful" do
+      before do
+        stub_request(:put, "#{list_member_url}/#{Digest::MD5.hexdigest(records.first[:email].downcase)}")
+          .to_return(status: 200, body: '{"detail": "Success"}', headers: {})
+      end
+
+      it "increments the success count" do
+        response = client.write(sync_config, records)
+
+        expect(response.tracking.success).to eq(records.size)
+        expect(response.tracking.failed).to eq(0)
+        log_message = response.tracking.logs.first
+        expect(log_message).to be_a(Multiwoven::Integrations::Protocol::LogMessage)
+        expect(log_message.level).to eql("info")
+        expect(log_message.message).to include("request")
+        expect(log_message.message).to include("response")
+      end
+    end
+
+    context "when the write operation fails" do
+      before do
+        # Mock the request to simulate a failure when trying to add/update a list member
+        stub_request(:put, "#{list_member_url}/#{Digest::MD5.hexdigest(records.first[:email].downcase)}")
+          .to_return(status: 400, body: '{"detail": "Invalid Request"}', headers: {})
+      end
+
+      it "increments the failure count" do
+        response = client.write(sync_config, records)
+
+        expect(response.tracking.failed).to eq(records.size)
+        expect(response.tracking.success).to eq(0)
+        log_message = response.tracking.logs.first
+        expect(log_message).to be_a(Multiwoven::Integrations::Protocol::LogMessage)
+        expect(log_message.level).to eql("error")
+        expect(log_message.message).to include("request")
+        expect(log_message.message).to include("response")
+      end
+    end
+  end
+
+  describe "#meta_data" do
+    it "serves it github image url as icon" do
+      image_url = "https://raw.githubusercontent.com/Multiwoven/multiwoven/main/integrations/lib/multiwoven/integrations/destination/mailchimp/icon.svg"
+      expect(client.send(:meta_data)[:data][:icon]).to eq(image_url)
+    end
+  end
+
+  private
+
+  def sync_config
+    Multiwoven::Integrations::Protocol::SyncConfig.from_json(
+      sync_config_json.to_json
+    )
+  end
+
+  describe "#discover" do
+    it "returns a catalog" do
+      message = client.discover
+      catalog = message.catalog
+      expect(catalog).to be_a(Multiwoven::Integrations::Protocol::Catalog)
+      catalog.streams.each do |stream|
+        case stream.name
+        when "Audience"
+          expect(stream.supported_sync_modes).to eql(["incremental"])
+        when "Tags"
+          expect(stream.supported_sync_modes).to eql(["incremental"])
+        when "Campaigns"
+          expect(stream.supported_sync_modes).to eql(["full_refresh"])
+        end
+      end
+    end
+  end
+end

From 6ecba38987aac0ec188adf0eeddf90517fbdc07a Mon Sep 17 00:00:00 2001
From: "github-actions[bot]"
 <41898282+github-actions[bot]@users.noreply.github.com>
Date: Tue, 12 Nov 2024 10:31:30 +0530
Subject: [PATCH 24/48] fix(CE): data app deletion error fix

Co-authored-by: afthab vp <afthab@multiwoven.com>
---
 server/app/models/data_app.rb                 | 2 +-
 server/app/serializers/feedback_serializer.rb | 6 ++++++
 server/spec/models/data_app_spec.rb           | 2 +-
 3 files changed, 8 insertions(+), 2 deletions(-)
 create mode 100644 server/app/serializers/feedback_serializer.rb

diff --git a/server/app/models/data_app.rb b/server/app/models/data_app.rb
index bd9c32a8..9dc3f2ff 100644
--- a/server/app/models/data_app.rb
+++ b/server/app/models/data_app.rb
@@ -13,7 +13,7 @@ class DataApp < ApplicationRecord
   has_many :visual_components, dependent: :destroy
   has_many :models, through: :visual_components
   has_many :feedbacks, through: :visual_components
-  has_many :data_app_sessions, dependent: :nullify
+  has_many :data_app_sessions, dependent: :destroy
 
   after_initialize :set_default_status, if: :new_record?
 
diff --git a/server/app/serializers/feedback_serializer.rb b/server/app/serializers/feedback_serializer.rb
new file mode 100644
index 00000000..1cb565f3
--- /dev/null
+++ b/server/app/serializers/feedback_serializer.rb
@@ -0,0 +1,6 @@
+# frozen_string_literal: true
+
+class FeedbackSerializer < ActiveModel::Serializer
+  attributes :id, :workspace_id, :data_app_id, :visual_component_id, :model_id, :reaction,
+             :feedback_content, :created_at, :updated_at, :feedback_type, :session_id, :additional_remarks
+end
diff --git a/server/spec/models/data_app_spec.rb b/server/spec/models/data_app_spec.rb
index 6cf1afdd..f105e912 100644
--- a/server/spec/models/data_app_spec.rb
+++ b/server/spec/models/data_app_spec.rb
@@ -14,7 +14,7 @@
     it { should belong_to(:workspace) }
     it { should have_many(:visual_components).dependent(:destroy) }
     it { should have_many(:models).through(:visual_components) }
-    it { should have_many(:data_app_sessions).dependent(:nullify) }
+    it { should have_many(:data_app_sessions).dependent(:destroy) }
     it { should have_many(:feedbacks).through(:visual_components) }
   end
 

From ba6d8df70dc010561379c1751ac898b2e9d8b512 Mon Sep 17 00:00:00 2001
From: afthab vp <afthab@multiwoven.com>
Date: Tue, 12 Nov 2024 10:59:17 +0530
Subject: [PATCH 25/48] fix(CE): mailchimp dependency build issue

---
 integrations/Gemfile.lock                           | 2 +-
 integrations/lib/multiwoven/integrations/rollout.rb | 2 +-
 integrations/multiwoven-integrations.gemspec        | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/integrations/Gemfile.lock b/integrations/Gemfile.lock
index a510a347..5794a858 100644
--- a/integrations/Gemfile.lock
+++ b/integrations/Gemfile.lock
@@ -7,7 +7,7 @@ GIT
 PATH
   remote: .
   specs:
-    multiwoven-integrations (0.12.0)
+    multiwoven-integrations (0.13.1)
       activesupport
       async-websocket
       aws-sdk-athena
diff --git a/integrations/lib/multiwoven/integrations/rollout.rb b/integrations/lib/multiwoven/integrations/rollout.rb
index b2f6dd7f..667eb292 100644
--- a/integrations/lib/multiwoven/integrations/rollout.rb
+++ b/integrations/lib/multiwoven/integrations/rollout.rb
@@ -2,7 +2,7 @@
 
 module Multiwoven
   module Integrations
-    VERSION = "0.13.0"
+    VERSION = "0.13.1"
 
     ENABLED_SOURCES = %w[
       Snowflake
diff --git a/integrations/multiwoven-integrations.gemspec b/integrations/multiwoven-integrations.gemspec
index 8a4174d1..c38ef063 100644
--- a/integrations/multiwoven-integrations.gemspec
+++ b/integrations/multiwoven-integrations.gemspec
@@ -52,6 +52,7 @@ Gem::Specification.new do |spec|
   spec.add_runtime_dependency "grpc"
   spec.add_runtime_dependency "hubspot-api-client"
   spec.add_runtime_dependency "iterable-api-client"
+  spec.add_runtime_dependency "MailchimpMarketing"
   spec.add_runtime_dependency "net-sftp"
   spec.add_runtime_dependency "pg"
   spec.add_runtime_dependency "rake"
@@ -67,7 +68,6 @@ Gem::Specification.new do |spec|
   spec.add_runtime_dependency "zendesk_api"
 
   spec.add_development_dependency "byebug"
-  spec.add_development_dependency "MailchimpMarketing"
   spec.add_development_dependency "rspec"
   spec.add_development_dependency "rubocop"
   spec.add_development_dependency "simplecov"

From a70219e029d0deabb1fbd8ca1c8eee7765c4fdd6 Mon Sep 17 00:00:00 2001
From: "Rafael E. O'Neill"
 <106079170+RafaelOAiSquared@users.noreply.github.com>
Date: Tue, 12 Nov 2024 08:22:09 -0400
Subject: [PATCH 26/48] Multiwoven release v0.33.0 (#459)

Co-authored-by: github-actions <developer@multiwoven.com>
---
 release-notes.md | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/release-notes.md b/release-notes.md
index d9c649fc..7f136c91 100644
--- a/release-notes.md
+++ b/release-notes.md
@@ -2,14 +2,14 @@
 
 All notable changes to this project will be documented in this file.
 
-## [0.32.0] - 2024-10-30
+## [0.33.0] - 2024-11-11
+
+### 🚀 Features
+
+- *(CE)* Added Mailchimp Destination Connector (#437)
 
 ### ⚙️ Miscellaneous Tasks
 
-- *(CE)* Add audit logs to report controller (#446)
-- *(CE)* Data app report model changes (#445)
-- *(CE)* Add audit logs to users controller (#449)
-- *(CE)* Add audit to sync records controller (#448)
-- *(CE)* Add audit logs to sync runs controller (#450)
+- *(CE)* Add additional_remarks in feedback model (#454)
 
 <!-- generated by git-cliff -->

From 61edf7c92f5b600fabcf5860bab02b7851feff1e Mon Sep 17 00:00:00 2001
From: afthab vp <afthab@multiwoven.com>
Date: Tue, 12 Nov 2024 19:40:48 +0530
Subject: [PATCH 27/48] chore(CE): add icons for ais data store and http model
 (#460)

---
 .../integrations/destination/ais_data_store/icon.svg     | 4 ++++
 .../multiwoven/integrations/source/http_model/icon.svg   | 9 +++++++++
 2 files changed, 13 insertions(+)
 create mode 100644 integrations/lib/multiwoven/integrations/destination/ais_data_store/icon.svg
 create mode 100644 integrations/lib/multiwoven/integrations/source/http_model/icon.svg

diff --git a/integrations/lib/multiwoven/integrations/destination/ais_data_store/icon.svg b/integrations/lib/multiwoven/integrations/destination/ais_data_store/icon.svg
new file mode 100644
index 00000000..277ee307
--- /dev/null
+++ b/integrations/lib/multiwoven/integrations/destination/ais_data_store/icon.svg
@@ -0,0 +1,4 @@
+<svg width="66" height="64" viewBox="0 0 66 64" fill="none" xmlns="http://www.w3.org/2000/svg">
+<path d="M24.5072 18.9904C26.4821 18.9904 28.2101 20.0795 29.1976 21.6527H51.2916V18.2643C51.2916 15.3599 48.9464 13.0607 45.9841 13.0607H18.2123C15.4968 13.0607 13.3985 15.2389 13.3985 17.7802V33.1491C13.3985 34.9643 14.8796 36.4165 16.7311 36.4165H34.9988C35.8628 34.4802 37.8377 33.1491 40.0595 33.1491C43.1452 33.1491 45.7373 35.6904 45.7373 38.8368C45.7373 41.9831 43.1452 44.5244 40.0595 44.5244C38.4549 44.5244 36.9737 43.7984 35.8628 42.5882H13.1516V46.5817C13.1516 49.002 15.1265 50.9382 17.5951 50.9382H47.8356C49.687 50.9382 51.1682 49.486 51.0448 47.6708V32.181C51.0448 30.0027 49.1933 28.1875 46.9716 28.1875H28.9507C27.9633 29.5186 26.3587 30.3657 24.5072 30.3657C21.4215 30.3657 18.8294 27.8244 18.8294 24.6781C18.8294 21.5317 21.4215 18.9904 24.5072 18.9904Z" fill="#00249C"/>
+<path d="M57.0292 0H8.02359C3.57976 0 0 3.54198 0 7.93893V56.0611C0 60.458 3.57976 64 8.02359 64H57.0292C61.4731 64 65.0528 60.458 65.0528 56.0611V7.93893C65.1763 3.54198 61.4731 0 57.0292 0ZM59.1277 51.6641C59.1277 55.2061 56.1652 58.1374 52.5854 58.1374H12.4674C8.88767 58.1374 5.92512 55.2061 5.92512 51.6641V12.3359C5.92512 8.79389 8.88767 5.8626 12.4674 5.8626H52.7088C56.2886 5.8626 59.2512 8.79389 59.2512 12.3359V51.6641H59.1277Z" fill="#00249C"/>
+</svg>
diff --git a/integrations/lib/multiwoven/integrations/source/http_model/icon.svg b/integrations/lib/multiwoven/integrations/source/http_model/icon.svg
new file mode 100644
index 00000000..d14a47e2
--- /dev/null
+++ b/integrations/lib/multiwoven/integrations/source/http_model/icon.svg
@@ -0,0 +1,9 @@
+<?xml version="1.0" encoding="utf-8"?>
+
<!-- Uploaded to: SVG Repo, www.svgrepo.com, Generator: SVG Repo Mixer Tools -->
+<svg width="800px" height="800px" viewBox="0 0 16 16" xmlns="http://www.w3.org/2000/svg" fill="none">
+
<g fill="#000000">
+
<path d="M6.032 4.132c0-1.039.843-1.882 1.886-1.882.709 0 1.328.39 1.65.97a.75.75 0 101.311-.728A3.385 3.385 0 007.918.75a3.383 3.383 0 00-3.386 3.382c0 .94.385 1.79 1.003 2.402l-2.054 3.594a1.25 1.25 0 00.151 2.49h.007a1.25 1.25 0 001.146-1.75l2.369-4.144a.75.75 0 00-.249-1.005 1.879 1.879 0 01-.873-1.587z"/>
+
<path d="M7.793 5.375a1.25 1.25 0 01.125-2.494h.006a1.25 1.25 0 011.157 1.725l2.159 3.572a3.383 3.383 0 014.51 3.19 3.383 3.383 0 01-4.23 3.275.75.75 0 11.373-1.452 1.883 1.883 0 002.357-1.822 1.883 1.883 0 00-2.897-1.588.75.75 0 01-1.045-.245l-2.515-4.16z"/>
+
<path d="M2.002 10.429a.75.75 0 00-1.298-.752 3.383 3.383 0 002.932 5.073c1.61 0 2.96-1.124 3.301-2.632h4.42c.229.304.592.5 1.001.5h.007a1.25 1.25 0 000-2.5h-.007c-.409 0-.772.197-1 .5H6.271a.75.75 0 00-.75.75 1.883 1.883 0 01-1.886 1.882 1.883 1.883 0 01-1.633-2.821z"/>
+
</g>
+
</svg>
\ No newline at end of file

From b276d1ceda80a45ca301a493f691cb5fae017c30 Mon Sep 17 00:00:00 2001
From: "github-actions[bot]"
 <41898282+github-actions[bot]@users.noreply.github.com>
Date: Wed, 13 Nov 2024 14:09:49 +0530
Subject: [PATCH 28/48] chore(CE): add timeout to http client (#458)

Co-authored-by: TivonB-AI2 <tivon.brown@squared.ai>
---
 integrations/Gemfile.lock                            |  2 +-
 .../lib/multiwoven/integrations/core/http_client.rb  |  7 ++++++-
 integrations/lib/multiwoven/integrations/rollout.rb  |  2 +-
 .../multiwoven/integrations/core/http_client_spec.rb | 12 ++++++++++++
 4 files changed, 20 insertions(+), 3 deletions(-)

diff --git a/integrations/Gemfile.lock b/integrations/Gemfile.lock
index 5794a858..a1944352 100644
--- a/integrations/Gemfile.lock
+++ b/integrations/Gemfile.lock
@@ -7,7 +7,7 @@ GIT
 PATH
   remote: .
   specs:
-    multiwoven-integrations (0.13.1)
+    multiwoven-integrations (0.13.2)
       activesupport
       async-websocket
       aws-sdk-athena
diff --git a/integrations/lib/multiwoven/integrations/core/http_client.rb b/integrations/lib/multiwoven/integrations/core/http_client.rb
index b119c256..c1e0939b 100644
--- a/integrations/lib/multiwoven/integrations/core/http_client.rb
+++ b/integrations/lib/multiwoven/integrations/core/http_client.rb
@@ -4,10 +4,15 @@ module Multiwoven
   module Integrations::Core
     class HttpClient
       class << self
-        def request(url, method, payload: nil, headers: {})
+        def request(url, method, payload: nil, headers: {}, config: {})
           uri = URI(url)
           http = Net::HTTP.new(uri.host, uri.port)
           http.use_ssl = (uri.scheme == "https")
+
+          # Set timeout if provided
+          http.open_timeout = config[:timeout] if config[:timeout]
+          http.read_timeout = config[:timeout] if config[:timeout]
+
           request = build_request(method, uri, payload, headers)
           http.request(request)
         end
diff --git a/integrations/lib/multiwoven/integrations/rollout.rb b/integrations/lib/multiwoven/integrations/rollout.rb
index 667eb292..ce530d91 100644
--- a/integrations/lib/multiwoven/integrations/rollout.rb
+++ b/integrations/lib/multiwoven/integrations/rollout.rb
@@ -2,7 +2,7 @@
 
 module Multiwoven
   module Integrations
-    VERSION = "0.13.1"
+    VERSION = "0.13.2"
 
     ENABLED_SOURCES = %w[
       Snowflake
diff --git a/integrations/spec/multiwoven/integrations/core/http_client_spec.rb b/integrations/spec/multiwoven/integrations/core/http_client_spec.rb
index b2d2d328..1f0d2460 100644
--- a/integrations/spec/multiwoven/integrations/core/http_client_spec.rb
+++ b/integrations/spec/multiwoven/integrations/core/http_client_spec.rb
@@ -51,6 +51,18 @@ module Integrations::Core
             expect { described_class.request(url, "INVALID", headers: headers) }.to raise_error(ArgumentError)
           end
         end
+
+        context "when timeout is set" do
+          it "raises a Net::OpenTimeout error if the request exceeds the timeout" do
+            stub_request(:get, url).to_timeout
+            expect { described_class.request(url, "GET", headers: headers, config: { timeout: 1 }) }.to raise_error(Net::OpenTimeout)
+          end
+
+          it "raises a Net::ReadTimeout error if the response exceeds the timeout" do
+            stub_request(:get, url).to_raise(Net::ReadTimeout)
+            expect { described_class.request(url, "GET", headers: headers, config: { timeout: 1 }) }.to raise_error(Net::ReadTimeout)
+          end
+        end
       end
     end
   end

From 2fd27d0e0a8b1dd283cb67c5a91027e23efe07eb Mon Sep 17 00:00:00 2001
From: "github-actions[bot]"
 <41898282+github-actions[bot]@users.noreply.github.com>
Date: Wed, 13 Nov 2024 14:30:15 +0530
Subject: [PATCH 29/48] feat(CE): Add HTTP Model Source Connector (#466)

Co-authored-by: TivonB-AI2 <124182151+TivonB-AI2@users.noreply.github.com>
---
 integrations/Gemfile.lock                     |   3 +-
 integrations/lib/multiwoven/integrations.rb   |   1 +
 .../lib/multiwoven/integrations/rollout.rb    |   3 +-
 .../integrations/source/http_model/client.rb  |  89 ++++++++
 .../source/http_model/config/catalog.json     |   6 +
 .../source/http_model/config/meta.json        |  15 ++
 .../source/http_model/config/spec.json        |  37 ++++
 .../source/http_model/client_spec.rb          | 192 ++++++++++++++++++
 8 files changed, 344 insertions(+), 2 deletions(-)
 create mode 100644 integrations/lib/multiwoven/integrations/source/http_model/client.rb
 create mode 100644 integrations/lib/multiwoven/integrations/source/http_model/config/catalog.json
 create mode 100644 integrations/lib/multiwoven/integrations/source/http_model/config/meta.json
 create mode 100644 integrations/lib/multiwoven/integrations/source/http_model/config/spec.json
 create mode 100644 integrations/spec/multiwoven/integrations/source/http_model/client_spec.rb

diff --git a/integrations/Gemfile.lock b/integrations/Gemfile.lock
index a1944352..57f8f681 100644
--- a/integrations/Gemfile.lock
+++ b/integrations/Gemfile.lock
@@ -7,7 +7,8 @@ GIT
 PATH
   remote: .
   specs:
-    multiwoven-integrations (0.13.2)
+    multiwoven-integrations (0.14.0)
+      MailchimpMarketing
       activesupport
       async-websocket
       aws-sdk-athena
diff --git a/integrations/lib/multiwoven/integrations.rb b/integrations/lib/multiwoven/integrations.rb
index d64b7d96..ea783a62 100644
--- a/integrations/lib/multiwoven/integrations.rb
+++ b/integrations/lib/multiwoven/integrations.rb
@@ -70,6 +70,7 @@
 require_relative "integrations/source/databrics_model/client"
 require_relative "integrations/source/aws_sagemaker_model/client"
 require_relative "integrations/source/google_vertex_model/client"
+require_relative "integrations/source/http_model/client"
 
 # Destination
 require_relative "integrations/destination/klaviyo/client"
diff --git a/integrations/lib/multiwoven/integrations/rollout.rb b/integrations/lib/multiwoven/integrations/rollout.rb
index ce530d91..26ea3a0e 100644
--- a/integrations/lib/multiwoven/integrations/rollout.rb
+++ b/integrations/lib/multiwoven/integrations/rollout.rb
@@ -2,7 +2,7 @@
 
 module Multiwoven
   module Integrations
-    VERSION = "0.13.2"
+    VERSION = "0.14.0"
 
     ENABLED_SOURCES = %w[
       Snowflake
@@ -19,6 +19,7 @@ module Integrations
       DatabricksModel
       AwsSagemakerModel
       VertexModel
+      HttpModel
     ].freeze
 
     ENABLED_DESTINATIONS = %w[
diff --git a/integrations/lib/multiwoven/integrations/source/http_model/client.rb b/integrations/lib/multiwoven/integrations/source/http_model/client.rb
new file mode 100644
index 00000000..9d175c89
--- /dev/null
+++ b/integrations/lib/multiwoven/integrations/source/http_model/client.rb
@@ -0,0 +1,89 @@
+# frozen_string_literal: true
+
+module Multiwoven::Integrations::Source
+  module HttpModel
+    include Multiwoven::Integrations::Core
+    class Client < SourceConnector
+      def check_connection(connection_config)
+        connection_config = connection_config.with_indifferent_access
+        url_host = connection_config[:url_host]
+        headers = connection_config[:headers]
+        response = Multiwoven::Integrations::Core::HttpClient.request(
+          url_host,
+          HTTP_GET,
+          headers: headers
+        )
+        if success?(response)
+          success_status
+        else
+          failure_status(nil)
+        end
+      rescue StandardError => e
+        handle_exception(e, {
+                           context: "HTTP MODEL:CHECK_CONNECTION:EXCEPTION",
+                           type: "error"
+                         })
+        failure_status(e)
+      end
+
+      def discover(_connection_config = nil)
+        catalog_json = read_json(CATALOG_SPEC_PATH)
+        catalog = build_catalog(catalog_json)
+        catalog.to_multiwoven_message
+      rescue StandardError => e
+        handle_exception(e, {
+                           context: "HTTP MODEL:DISCOVER:EXCEPTION",
+                           type: "error"
+                         })
+      end
+
+      def read(sync_config)
+        connection_config = sync_config.source.connection_specification
+        connection_config = connection_config.with_indifferent_access
+        # The server checks the ConnectorQueryType.
+        # If it's "ai_ml," the server calculates the payload and passes it as a query in the sync config model protocol.
+        # This query is then sent to the AI/ML model.
+        payload = JSON.parse(sync_config.model.query)
+        run_model(connection_config, payload)
+      rescue StandardError => e
+        handle_exception(e, {
+                           context: "HTTP MODEL:READ:EXCEPTION",
+                           type: "error"
+                         })
+      end
+
+      private
+
+      def run_model(connection_config, payload)
+        connection_config = connection_config.with_indifferent_access
+        url_host = connection_config[:url_host]
+        headers = connection_config[:headers]
+        config = connection_config[:config]
+        config[:timeout] ||= 30
+        response = send_request(url_host, payload, headers, config)
+        process_response(response)
+      rescue StandardError => e
+        handle_exception(e, context: "HTTP MODEL:RUN_MODEL:EXCEPTION", type: "error")
+      end
+
+      def process_response(response)
+        if success?(response)
+          data = JSON.parse(response.body)
+          [RecordMessage.new(data: data, emitted_at: Time.now.to_i).to_multiwoven_message]
+        else
+          create_log_message("HTTP MODEL:RUN_MODEL", "error", "request failed")
+        end
+      end
+
+      def send_request(url, payload, headers, config)
+        Multiwoven::Integrations::Core::HttpClient.request(
+          url,
+          HTTP_POST,
+          payload: payload,
+          headers: headers,
+          config: config
+        )
+      end
+    end
+  end
+end
diff --git a/integrations/lib/multiwoven/integrations/source/http_model/config/catalog.json b/integrations/lib/multiwoven/integrations/source/http_model/config/catalog.json
new file mode 100644
index 00000000..dacb788b
--- /dev/null
+++ b/integrations/lib/multiwoven/integrations/source/http_model/config/catalog.json
@@ -0,0 +1,6 @@
+{
+  "request_rate_limit": 600,
+  "request_rate_limit_unit": "minute",
+  "request_rate_concurrency": 10,
+  "streams": []
+}
diff --git a/integrations/lib/multiwoven/integrations/source/http_model/config/meta.json b/integrations/lib/multiwoven/integrations/source/http_model/config/meta.json
new file mode 100644
index 00000000..dca2f332
--- /dev/null
+++ b/integrations/lib/multiwoven/integrations/source/http_model/config/meta.json
@@ -0,0 +1,15 @@
+{
+  "data": {
+    "name": "HttpModel",
+    "title": "HTTP Model",
+    "connector_type": "source",
+    "category": "AI Model",
+    "documentation_url": "https://docs.mutliwoven.com",
+    "github_issue_label": "source-http-model",
+    "icon": "icon.svg",
+    "license": "MIT",
+    "release_stage": "alpha",
+    "support_level": "community",
+    "tags": ["language:ruby", "multiwoven"]
+  }
+}
diff --git a/integrations/lib/multiwoven/integrations/source/http_model/config/spec.json b/integrations/lib/multiwoven/integrations/source/http_model/config/spec.json
new file mode 100644
index 00000000..f270b35d
--- /dev/null
+++ b/integrations/lib/multiwoven/integrations/source/http_model/config/spec.json
@@ -0,0 +1,37 @@
+{
+  "documentation_url": "https://docs.multiwoven.com/integrations/source/http-model",
+  "stream_type": "user_defined",
+  "connector_query_type": "ai_ml",
+  "connection_specification": {
+    "$schema": "http://json-schema.org/draft-07/schema#",
+    "title": "Http Model",
+    "type": "object",
+    "required": ["url_host"],
+    "properties": {
+      "url_host": {
+        "type": "string",
+        "title": "URL",
+        "order": 0
+      },
+      "headers": {
+        "type": "string",
+        "title": "Http Headers",
+        "order": 1
+      },
+      "config": {
+        "title": "",
+        "type": "object",
+        "properties": {
+          "timeout": {
+            "type": "integer",
+            "minimum": 0,
+            "default": 30,
+            "title": "Http Timeout",
+            "order": 0
+          }
+        },
+        "order": 2
+      }
+    }
+  }
+}
diff --git a/integrations/spec/multiwoven/integrations/source/http_model/client_spec.rb b/integrations/spec/multiwoven/integrations/source/http_model/client_spec.rb
new file mode 100644
index 00000000..fb7dd951
--- /dev/null
+++ b/integrations/spec/multiwoven/integrations/source/http_model/client_spec.rb
@@ -0,0 +1,192 @@
+# frozen_string_literal: true
+
+RSpec.describe Multiwoven::Integrations::Source::HttpModel::Client do
+  include WebMock::API
+
+  before(:each) do
+    WebMock.disable_net_connect!(allow_localhost: true)
+  end
+
+  let(:client) { described_class.new }
+  let(:mock_http_session) { double("Net::Http::Session") }
+
+  let(:payload) do
+    {
+      queries: "Hello there"
+    }
+  end
+
+  let(:sync_config_json) do
+    {
+      source: {
+        name: "DestinationConnectorName",
+        type: "destination",
+        connection_specification: {
+          url_host: "https://your-subdomain",
+          headers: {
+            "Accept" => "application/json",
+            "Authorization" => "Bearer test_token",
+            "Content-Type" => "application/json"
+          },
+          config: {
+            timeout: 25
+          }
+        }
+      },
+      destination: {
+        name: "Http",
+        type: "destination",
+        connection_specification: {
+          example_destination_key: "example_destination_value"
+        }
+      },
+      model: {
+        name: "ExampleModel",
+        query: payload.to_json,
+        query_type: "ai_ml",
+        primary_key: "id"
+      },
+      stream: {
+        name: "example_stream",
+        json_schema: { "field1": "type1" },
+        request_method: "POST",
+        request_rate_limit: 4,
+        rate_limit_unit_seconds: 1
+      },
+      sync_mode: "full_refresh",
+      cursor_field: "timestamp",
+      destination_sync_mode: "upsert",
+      sync_id: "1"
+    }
+  end
+
+  let(:sync_config) { Multiwoven::Integrations::Protocol::SyncConfig.from_json(sync_config_json.to_json) }
+
+  before do
+    allow(Multiwoven::Integrations::Core::HttpClient).to receive(:request)
+  end
+
+  describe "#check_connection" do
+    context "when the connection is successful" do
+      let(:response_body) { { "message" => "success" }.to_json }
+      before do
+        response = Net::HTTPSuccess.new("1.1", "200", "Unauthorized")
+        response.content_type = "application/json"
+        url = sync_config_json[:source][:connection_specification][:url_host]
+        headers = sync_config_json[:source][:connection_specification][:headers]
+        allow(response).to receive(:body).and_return(response_body)
+        allow(Multiwoven::Integrations::Core::HttpClient).to receive(:request)
+          .with(url,
+                "GET",
+                headers: headers)
+          .and_return(response)
+      end
+
+      it "returns a successful connection status" do
+        response = client.check_connection(sync_config_json[:source][:connection_specification])
+        expect(response).to be_a(Multiwoven::Integrations::Protocol::MultiwovenMessage)
+        expect(response.connection_status.status).to eq("succeeded")
+      end
+    end
+
+    context "when the connection fails" do
+      let(:response_body) { { "message" => "failed" }.to_json }
+      before do
+        response = Net::HTTPSuccess.new("1.1", "401", "Unauthorized")
+        response.content_type = "application/json"
+        url = sync_config_json[:source][:connection_specification][:url_host]
+        headers = sync_config_json[:source][:connection_specification][:headers]
+        allow(response).to receive(:body).and_return(response_body)
+        allow(Multiwoven::Integrations::Core::HttpClient).to receive(:request)
+          .with(url,
+                "GET",
+                headers: headers)
+          .and_return(response)
+      end
+
+      it "returns a failed connection status with an error message" do
+        response = client.check_connection(sync_config_json[:source][:connection_specification])
+
+        expect(response).to be_a(Multiwoven::Integrations::Protocol::MultiwovenMessage)
+        expect(response.connection_status.status).to eq("failed")
+      end
+    end
+  end
+
+  describe "#discover" do
+    it "successfully returns the catalog message" do
+      message = client.discover(nil)
+      catalog = message.catalog
+      expect(catalog).to be_a(Multiwoven::Integrations::Protocol::Catalog)
+      expect(catalog.request_rate_limit).to eql(600)
+      expect(catalog.request_rate_limit_unit).to eql("minute")
+      expect(catalog.request_rate_concurrency).to eql(10)
+    end
+
+    it "handles exceptions during discovery" do
+      allow(client).to receive(:read_json).and_raise(StandardError.new("test error"))
+      expect(client).to receive(:handle_exception).with(
+        an_instance_of(StandardError),
+        hash_including(context: "HTTP MODEL:DISCOVER:EXCEPTION", type: "error")
+      )
+      client.discover
+    end
+  end
+
+  describe "#read" do
+    context "when the read is successful" do
+      let(:response_body) { { "message" => "Hello! how can I help" }.to_json }
+      before do
+        response = Net::HTTPSuccess.new("1.1", "200", "Unauthorized")
+        response.content_type = "application/json"
+        url = sync_config_json[:source][:connection_specification][:url_host]
+        headers = sync_config_json[:source][:connection_specification][:headers]
+        config = sync_config_json[:source][:connection_specification][:config]
+        allow(response).to receive(:body).and_return(response_body)
+        allow(Multiwoven::Integrations::Core::HttpClient).to receive(:request)
+          .with(url,
+                "POST",
+                payload: JSON.parse(payload.to_json),
+                headers: headers,
+                config: config)
+          .and_return(response)
+      end
+
+      it "successfully reads records" do
+        records = client.read(sync_config)
+        expect(records).to be_an(Array)
+        expect(records.first.record).to be_a(Multiwoven::Integrations::Protocol::RecordMessage)
+        expect(records.first.record.data).to eq(JSON.parse(response_body))
+      end
+    end
+
+    context "when the write operation fails" do
+      let(:response_body) { { "message" => "failed" }.to_json }
+      before do
+        response = Net::HTTPSuccess.new("1.1", "401", "Unauthorized")
+        response.content_type = "application/json"
+        url = sync_config_json[:source][:connection_specification][:url_host]
+        headers = sync_config_json[:source][:connection_specification][:headers]
+        config = sync_config_json[:source][:connection_specification][:config]
+        allow(response).to receive(:body).and_return(response_body)
+        allow(Multiwoven::Integrations::Core::HttpClient).to receive(:request)
+          .with(url,
+                "GET",
+                headers: headers,
+                config: config)
+          .and_return(response)
+      end
+
+      it "handles exceptions during reading" do
+        error_instance = StandardError.new("test error")
+        allow(client).to receive(:run_model).and_raise(error_instance)
+        expect(client).to receive(:handle_exception).with(
+          error_instance,
+          hash_including(context: "HTTP MODEL:READ:EXCEPTION", type: "error")
+        )
+
+        client.read(sync_config)
+      end
+    end
+  end
+end

From 9626b58a0c6fd80cf849e9ab5d344b0161ecb0f3 Mon Sep 17 00:00:00 2001
From: "github-actions[bot]"
 <41898282+github-actions[bot]@users.noreply.github.com>
Date: Wed, 13 Nov 2024 14:36:44 +0530
Subject: [PATCH 30/48] chore(CE): Dynamic sql model type added (#465)

Co-authored-by: Basil V Bose <bvb007@gmail.com>
---
 .../lib/multiwoven/integrations/protocol/protocol.rb         | 2 +-
 integrations/lib/multiwoven/integrations/rollout.rb          | 2 +-
 .../spec/multiwoven/integrations/protocol/protocol_spec.rb   | 5 +++++
 3 files changed, 7 insertions(+), 2 deletions(-)

diff --git a/integrations/lib/multiwoven/integrations/protocol/protocol.rb b/integrations/lib/multiwoven/integrations/protocol/protocol.rb
index 22fcdfbb..6595b619 100644
--- a/integrations/lib/multiwoven/integrations/protocol/protocol.rb
+++ b/integrations/lib/multiwoven/integrations/protocol/protocol.rb
@@ -11,7 +11,7 @@ module Types
     DestinationSyncMode = Types::String.enum("insert", "upsert")
     ConnectorType = Types::String.enum("source", "destination")
     ConnectorQueryType = Types::String.enum("raw_sql", "soql", "ai_ml")
-    ModelQueryType = Types::String.enum("raw_sql", "dbt", "soql", "table_selector", "ai_ml")
+    ModelQueryType = Types::String.enum("raw_sql", "dbt", "soql", "table_selector", "ai_ml", "dynamic_sql")
     ConnectionStatusType = Types::String.enum("succeeded", "failed")
     StreamType = Types::String.enum("static", "dynamic", "user_defined")
     StreamAction = Types::String.enum("fetch", "create", "update", "delete")
diff --git a/integrations/lib/multiwoven/integrations/rollout.rb b/integrations/lib/multiwoven/integrations/rollout.rb
index 26ea3a0e..2cb81bb7 100644
--- a/integrations/lib/multiwoven/integrations/rollout.rb
+++ b/integrations/lib/multiwoven/integrations/rollout.rb
@@ -2,7 +2,7 @@
 
 module Multiwoven
   module Integrations
-    VERSION = "0.14.0"
+    VERSION = "0.14.1"
 
     ENABLED_SOURCES = %w[
       Snowflake
diff --git a/integrations/spec/multiwoven/integrations/protocol/protocol_spec.rb b/integrations/spec/multiwoven/integrations/protocol/protocol_spec.rb
index f0d2c640..10192c99 100644
--- a/integrations/spec/multiwoven/integrations/protocol/protocol_spec.rb
+++ b/integrations/spec/multiwoven/integrations/protocol/protocol_spec.rb
@@ -383,6 +383,11 @@ module Integrations::Protocol
           model = Model.new(name: "Test", query: "SELECT * FROM table", query_type: "ai_ml", primary_key: "id")
           expect("ai_ml").to include(model.query_type)
         end
+
+        it "has a query_type 'dynamic_sql'" do
+          model = Model.new(name: "Test", query: "SELECT * FROM table", query_type: "dynamic_sql", primary_key: "id")
+          expect("dynamic_sql").to include(model.query_type)
+        end
       end
     end
 

From 8afc33840c20c16374c263895c4ccc72e7014898 Mon Sep 17 00:00:00 2001
From: "github-actions[bot]"
 <41898282+github-actions[bot]@users.noreply.github.com>
Date: Wed, 13 Nov 2024 14:44:24 +0530
Subject: [PATCH 31/48] fix(CE): http model config (#464)

Co-authored-by: afthab vp <afthab@multiwoven.com>
---
 integrations/Gemfile.lock                     |  2 +-
 .../lib/multiwoven/integrations/rollout.rb    |  2 +-
 .../source/http_model/config/spec.json        | 20 +++++++++++++++++--
 3 files changed, 20 insertions(+), 4 deletions(-)

diff --git a/integrations/Gemfile.lock b/integrations/Gemfile.lock
index 57f8f681..b268e97f 100644
--- a/integrations/Gemfile.lock
+++ b/integrations/Gemfile.lock
@@ -7,7 +7,7 @@ GIT
 PATH
   remote: .
   specs:
-    multiwoven-integrations (0.14.0)
+    multiwoven-integrations (0.14.2)
       MailchimpMarketing
       activesupport
       async-websocket
diff --git a/integrations/lib/multiwoven/integrations/rollout.rb b/integrations/lib/multiwoven/integrations/rollout.rb
index 2cb81bb7..4abff0f7 100644
--- a/integrations/lib/multiwoven/integrations/rollout.rb
+++ b/integrations/lib/multiwoven/integrations/rollout.rb
@@ -2,7 +2,7 @@
 
 module Multiwoven
   module Integrations
-    VERSION = "0.14.1"
+    VERSION = "0.14.2"
 
     ENABLED_SOURCES = %w[
       Snowflake
diff --git a/integrations/lib/multiwoven/integrations/source/http_model/config/spec.json b/integrations/lib/multiwoven/integrations/source/http_model/config/spec.json
index f270b35d..4d393ec9 100644
--- a/integrations/lib/multiwoven/integrations/source/http_model/config/spec.json
+++ b/integrations/lib/multiwoven/integrations/source/http_model/config/spec.json
@@ -14,9 +14,11 @@
         "order": 0
       },
       "headers": {
-        "type": "string",
         "title": "Http Headers",
-        "order": 1
+        "order": 1,
+        "additionalProperties": {
+          "type": "string"
+        }
       },
       "config": {
         "title": "",
@@ -31,6 +33,20 @@
           }
         },
         "order": 2
+      },
+      "request_format": {
+        "title": "Request Format",
+        "description": "Sample Request Format",
+        "type": "string",
+        "x-request-format": true,
+        "order": 3
+      },
+      "response_format": {
+        "title": "Response Format",
+        "description": "Sample Response Format",
+        "type": "string",
+        "x-response-format": true,
+        "order": 4
       }
     }
   }

From 47c2af950878e71d3be0cf20ec36194012248ea9 Mon Sep 17 00:00:00 2001
From: "github-actions[bot]"
 <41898282+github-actions[bot]@users.noreply.github.com>
Date: Wed, 13 Nov 2024 14:58:13 +0530
Subject: [PATCH 32/48] feat(CE): AIS Data store destination connector (#462)

Co-authored-by: afthab vp <afthab@multiwoven.com>
---
 integrations/Gemfile.lock                     |   2 +-
 integrations/lib/multiwoven/integrations.rb   |   1 +
 .../destination/ais_data_store/client.rb      | 127 +++++++++++
 .../ais_data_store/config/meta.json           |  15 ++
 .../ais_data_store/config/spec.json           |  68 ++++++
 .../destination/ais_data_store/icon.svg       |   7 +
 .../lib/multiwoven/integrations/rollout.rb    |   3 +-
 .../destination/ais_data_store/client_spec.rb | 215 ++++++++++++++++++
 8 files changed, 436 insertions(+), 2 deletions(-)
 create mode 100644 integrations/lib/multiwoven/integrations/destination/ais_data_store/client.rb
 create mode 100644 integrations/lib/multiwoven/integrations/destination/ais_data_store/config/meta.json
 create mode 100644 integrations/lib/multiwoven/integrations/destination/ais_data_store/config/spec.json
 create mode 100644 integrations/spec/multiwoven/integrations/destination/ais_data_store/client_spec.rb

diff --git a/integrations/Gemfile.lock b/integrations/Gemfile.lock
index b268e97f..80bacca5 100644
--- a/integrations/Gemfile.lock
+++ b/integrations/Gemfile.lock
@@ -7,7 +7,7 @@ GIT
 PATH
   remote: .
   specs:
-    multiwoven-integrations (0.14.2)
+    multiwoven-integrations (0.15.0)
       MailchimpMarketing
       activesupport
       async-websocket
diff --git a/integrations/lib/multiwoven/integrations.rb b/integrations/lib/multiwoven/integrations.rb
index ea783a62..f8164b95 100644
--- a/integrations/lib/multiwoven/integrations.rb
+++ b/integrations/lib/multiwoven/integrations.rb
@@ -93,6 +93,7 @@
 require_relative "integrations/destination/microsoft_excel/client"
 require_relative "integrations/destination/microsoft_sql/client"
 require_relative "integrations/destination/mailchimp/client"
+require_relative "integrations/destination/ais_data_store/client"
 
 module Multiwoven
   module Integrations
diff --git a/integrations/lib/multiwoven/integrations/destination/ais_data_store/client.rb b/integrations/lib/multiwoven/integrations/destination/ais_data_store/client.rb
new file mode 100644
index 00000000..67d878d6
--- /dev/null
+++ b/integrations/lib/multiwoven/integrations/destination/ais_data_store/client.rb
@@ -0,0 +1,127 @@
+# frozen_string_literal: true
+
+require "pg"
+
+module Multiwoven::Integrations::Destination
+  module AISDataStore
+    include Multiwoven::Integrations::Core
+    class Client < DestinationConnector
+      def check_connection(connection_config)
+        connection_config = connection_config.with_indifferent_access
+        create_connection(connection_config)
+        ConnectionStatus.new(
+          status: ConnectionStatusType["succeeded"]
+        ).to_multiwoven_message
+      rescue PG::Error => e
+        ConnectionStatus.new(
+          status: ConnectionStatusType["failed"], message: e.message
+        ).to_multiwoven_message
+      end
+
+      def discover(connection_config)
+        connection_config = connection_config.with_indifferent_access
+        query = "SELECT table_name, column_name, data_type, is_nullable
+                 FROM information_schema.columns
+                 WHERE table_schema = '#{connection_config[:schema]}' AND table_catalog = '#{connection_config[:database]}'
+                 ORDER BY table_name, ordinal_position;"
+
+        db = create_connection(connection_config)
+        records = db.exec(query) do |result|
+          result.map do |row|
+            row
+          end
+        end
+        catalog = Catalog.new(streams: create_streams(records))
+        catalog.to_multiwoven_message
+      rescue StandardError => e
+        handle_exception(e, {
+                           context: "POSTGRESQL:DISCOVER:EXCEPTION",
+                           type: "error"
+                         })
+      ensure
+        db&.close
+      end
+
+      def write(sync_config, records, action = "destination_insert")
+        connection_config = sync_config.destination.connection_specification.with_indifferent_access
+        table_name = sync_config.stream.name
+        primary_key = sync_config.model.primary_key
+        log_message_array = []
+        db = create_connection(connection_config)
+
+        write_success = 0
+        write_failure = 0
+
+        records.each do |record|
+          query = Multiwoven::Integrations::Core::QueryBuilder.perform(action, table_name, record, primary_key)
+          logger.debug("POSTGRESQL:WRITE:QUERY query = #{query} sync_id = #{sync_config.sync_id} sync_run_id = #{sync_config.sync_run_id}")
+          begin
+            response = db.exec(query)
+            write_success += 1
+            log_message_array << log_request_response("info", query, response)
+          rescue StandardError => e
+            handle_exception(e, {
+                               context: "POSTGRESQL:RECORD:WRITE:EXCEPTION",
+                               type: "error",
+                               sync_id: sync_config.sync_id,
+                               sync_run_id: sync_config.sync_run_id
+                             })
+            write_failure += 1
+            log_message_array << log_request_response("error", query, e.message)
+          end
+        end
+        tracking_message(write_success, write_failure, log_message_array)
+      rescue StandardError => e
+        handle_exception(e, {
+                           context: "POSTGRESQL:RECORD:WRITE:EXCEPTION",
+                           type: "error",
+                           sync_id: sync_config.sync_id,
+                           sync_run_id: sync_config.sync_run_id
+                         })
+      end
+
+      private
+
+      def query(connection, query)
+        connection.exec(query) do |result|
+          result.map do |row|
+            RecordMessage.new(data: row, emitted_at: Time.now.to_i).to_multiwoven_message
+          end
+        end
+      end
+
+      def create_connection(connection_config)
+        raise "Unsupported Auth type" unless connection_config[:credentials][:auth_type] == "username/password"
+
+        PG.connect(
+          host: connection_config[:host],
+          dbname: connection_config[:database],
+          user: connection_config[:credentials][:username],
+          password: connection_config[:credentials][:password],
+          port: connection_config[:port]
+        )
+      end
+
+      def create_streams(records)
+        group_by_table(records).map do |r|
+          Multiwoven::Integrations::Protocol::Stream.new(name: r[:tablename], action: StreamAction["fetch"], json_schema: convert_to_json_schema(r[:columns]))
+        end
+      end
+
+      def group_by_table(records)
+        records.group_by { |entry| entry["table_name"] }.map do |table_name, columns|
+          {
+            tablename: table_name,
+            columns: columns.map do |column|
+              {
+                column_name: column["column_name"],
+                type: column["data_type"],
+                optional: column["is_nullable"] == "YES"
+              }
+            end
+          }
+        end
+      end
+    end
+  end
+end
diff --git a/integrations/lib/multiwoven/integrations/destination/ais_data_store/config/meta.json b/integrations/lib/multiwoven/integrations/destination/ais_data_store/config/meta.json
new file mode 100644
index 00000000..194991f1
--- /dev/null
+++ b/integrations/lib/multiwoven/integrations/destination/ais_data_store/config/meta.json
@@ -0,0 +1,15 @@
+{
+  "data": {
+    "name": "AISDataStore",
+    "title": "AIS Data Store",
+    "connector_type": "destination",
+    "category": "Database",
+    "documentation_url": "https://docs.mutliwoven.com",
+    "github_issue_label": "destination-postgresql",
+    "icon": "icon.svg",
+    "license": "MIT",
+    "release_stage": "alpha",
+    "support_level": "community",
+    "tags": ["language:ruby", "multiwoven"]
+  }
+}
diff --git a/integrations/lib/multiwoven/integrations/destination/ais_data_store/config/spec.json b/integrations/lib/multiwoven/integrations/destination/ais_data_store/config/spec.json
new file mode 100644
index 00000000..7ce099e6
--- /dev/null
+++ b/integrations/lib/multiwoven/integrations/destination/ais_data_store/config/spec.json
@@ -0,0 +1,68 @@
+{
+  "documentation_url": "https://docs.multiwoven.com/integrations/sources/postgresql",
+  "stream_type": "dynamic",
+  "connection_specification": {
+    "$schema": "http://json-schema.org/draft-07/schema#",
+    "title": "Postgresql",
+    "type": "object",
+    "required": ["host", "port", "database", "schema"],
+    "properties": {
+      "credentials": {
+        "title": "",
+        "type": "object",
+        "required": ["auth_type", "username", "password"],
+        "properties": {
+          "auth_type": {
+            "type": "string",
+            "default": "username/password",
+            "order": 0,
+            "readOnly": true
+          },
+          "username": {
+            "description": "Username refers to your individual PostgreSQL login credentials. At a minimum, the user associated with these credentials must be granted read access to the data intended for synchronization.",
+            "examples": ["POSTGRESQL_USER"],
+            "type": "string",
+            "title": "Username",
+            "order": 1
+          },
+          "password": {
+            "description": "This field requires the password associated with the user account specified in the preceding section.",
+            "type": "string",
+            "multiwoven_secret": true,
+            "title": "Password",
+            "order": 2
+          }
+        },
+        "order": 0
+      },
+      "host": {
+        "description": "The hostname or IP address of your PostgreSQL server.",
+        "examples": ["127.0.0.1"],
+        "type": "string",
+        "title": "Host",
+        "order": 1
+      },
+      "port": {
+        "description": "The port number for your PostgreSQL server, which defaults to 5432, may vary based on your configuration. ",
+        "examples": ["5432"],
+        "type": "string",
+        "title": "Port",
+        "order": 2
+      },
+      "database": {
+        "description": "The specific PostgreSQL database to connect to.",
+        "examples": ["POSTGRESQL_DB"],
+        "type": "string",
+        "title": "Database",
+        "order": 3
+      },
+      "schema": {
+        "description": "The schema within the PostgreSQL database.",
+        "examples": ["POSTGRESQL_SCHEMA"],
+        "type": "string",
+        "title": "Schema",
+        "order": 4
+      }
+    }
+  }
+}
diff --git a/integrations/lib/multiwoven/integrations/destination/ais_data_store/icon.svg b/integrations/lib/multiwoven/integrations/destination/ais_data_store/icon.svg
index 277ee307..eb2d575c 100644
--- a/integrations/lib/multiwoven/integrations/destination/ais_data_store/icon.svg
+++ b/integrations/lib/multiwoven/integrations/destination/ais_data_store/icon.svg
@@ -1,4 +1,11 @@
+<<<<<<< HEAD
 <svg width="66" height="64" viewBox="0 0 66 64" fill="none" xmlns="http://www.w3.org/2000/svg">
 <path d="M24.5072 18.9904C26.4821 18.9904 28.2101 20.0795 29.1976 21.6527H51.2916V18.2643C51.2916 15.3599 48.9464 13.0607 45.9841 13.0607H18.2123C15.4968 13.0607 13.3985 15.2389 13.3985 17.7802V33.1491C13.3985 34.9643 14.8796 36.4165 16.7311 36.4165H34.9988C35.8628 34.4802 37.8377 33.1491 40.0595 33.1491C43.1452 33.1491 45.7373 35.6904 45.7373 38.8368C45.7373 41.9831 43.1452 44.5244 40.0595 44.5244C38.4549 44.5244 36.9737 43.7984 35.8628 42.5882H13.1516V46.5817C13.1516 49.002 15.1265 50.9382 17.5951 50.9382H47.8356C49.687 50.9382 51.1682 49.486 51.0448 47.6708V32.181C51.0448 30.0027 49.1933 28.1875 46.9716 28.1875H28.9507C27.9633 29.5186 26.3587 30.3657 24.5072 30.3657C21.4215 30.3657 18.8294 27.8244 18.8294 24.6781C18.8294 21.5317 21.4215 18.9904 24.5072 18.9904Z" fill="#00249C"/>
 <path d="M57.0292 0H8.02359C3.57976 0 0 3.54198 0 7.93893V56.0611C0 60.458 3.57976 64 8.02359 64H57.0292C61.4731 64 65.0528 60.458 65.0528 56.0611V7.93893C65.1763 3.54198 61.4731 0 57.0292 0ZM59.1277 51.6641C59.1277 55.2061 56.1652 58.1374 52.5854 58.1374H12.4674C8.88767 58.1374 5.92512 55.2061 5.92512 51.6641V12.3359C5.92512 8.79389 8.88767 5.8626 12.4674 5.8626H52.7088C56.2886 5.8626 59.2512 8.79389 59.2512 12.3359V51.6641H59.1277Z" fill="#00249C"/>
+=======
+<?xml version="1.0" encoding="UTF-8"?>
+<svg version="1.1" xmlns="http://www.w3.org/2000/svg" width="120" height="120">
+<path d="M0 0 C0.86770523 -0.01004562 1.73541046 -0.02009125 2.62940979 -0.03044128 C3.58369873 -0.03627228 4.53798767 -0.04210327 5.52119446 -0.04811096 C6.5246138 -0.05789474 7.52803314 -0.06767853 8.56185913 -0.07775879 C11.89374965 -0.10775492 15.22559986 -0.12911119 18.55757141 -0.14796448 C20.26279433 -0.15782222 20.26279433 -0.15782222 22.0024662 -0.1678791 C28.02106609 -0.20049291 34.03964812 -0.22493756 40.05830383 -0.24440002 C45.04131327 -0.26221014 50.02375652 -0.29702378 55.00654602 -0.34693909 C61.01949684 -0.4071084 67.03192274 -0.43787191 73.04516029 -0.44757271 C75.3365012 -0.45613508 77.62782781 -0.47497223 79.91899681 -0.50431633 C83.12946569 -0.54321656 86.33788969 -0.54426658 89.54853821 -0.53639221 C90.96478989 -0.56639313 90.96478989 -0.56639313 92.40965271 -0.59700012 C98.27116391 -0.53256805 101.35293046 0.57580113 105.59190083 4.67732143 C107.78589597 7.24699689 107.98906384 9.01053142 108.01898193 12.30227661 C108.02902756 13.16998184 108.03907318 14.03768707 108.04942322 14.9316864 C108.05525421 15.88597534 108.06108521 16.84026428 108.0670929 17.82347107 C108.07687668 18.82689041 108.08666046 19.83030975 108.09674072 20.86413574 C108.12673686 24.19602626 108.14809313 27.52787647 108.16694641 30.85984802 C108.17351824 31.9966633 108.18009007 33.13347858 108.18686104 34.30474281 C108.21947484 40.3233427 108.24391949 46.34192473 108.26338196 52.36058044 C108.28119207 57.34358989 108.31600571 62.32603313 108.36592102 67.30882263 C108.42609034 73.32177345 108.45685385 79.33419935 108.46655464 85.3474369 C108.47511702 87.63877781 108.49395416 89.93010442 108.52329826 92.22127342 C108.5621985 95.43174231 108.56324851 98.6401663 108.55537415 101.85081482 C108.57537476 102.7949826 108.59537537 103.73915039 108.61598206 104.71192932 C108.55154998 110.57344052 107.44318081 113.65520707 103.3416605 117.89417744 C100.77198505 120.08817258 99.00845051 120.29134045 95.71670532 120.32125854 C94.84900009 120.33130417 93.98129486 120.34134979 93.08729553 120.35169983 C91.65586212 120.36044632 91.65586212 120.36044632 90.19551086 120.36936951 C89.19209152 120.37915329 88.18867218 120.38893707 87.15484619 120.39901733 C83.82295567 120.42901347 80.49110546 120.45036974 77.15913391 120.46922302 C75.45391099 120.47908076 75.45391099 120.47908076 73.71423912 120.48913765 C67.69563923 120.52175145 61.6770572 120.5461961 55.65840149 120.56565857 C50.67539205 120.58346868 45.69294881 120.61828232 40.7101593 120.66819763 C34.69720848 120.72836695 28.68478258 120.75913046 22.67154503 120.76883125 C20.38020412 120.77739363 18.08887751 120.79623078 15.79770851 120.82557487 C12.58723963 120.86447511 9.37881563 120.86552512 6.16816711 120.85765076 C4.75191544 120.88765167 4.75191544 120.88765167 3.30705261 120.91825867 C-2.55445859 120.85382659 -5.63622513 119.74545742 -9.8751955 115.64393711 C-12.06919065 113.07426166 -12.27235852 111.31072713 -12.30227661 108.01898193 C-12.31232224 107.1512767 -12.32236786 106.28357147 -12.3327179 105.38957214 C-12.34146439 103.95813873 -12.34146439 103.95813873 -12.35038757 102.49778748 C-12.36017136 101.49436813 -12.36995514 100.49094879 -12.3800354 99.4571228 C-12.41003153 96.12523229 -12.4313878 92.79338207 -12.45024109 89.46141052 C-12.45681292 88.32459524 -12.46338474 87.18777996 -12.47015572 86.01651573 C-12.50276952 79.99791585 -12.52721417 73.97933381 -12.54667664 67.9606781 C-12.56448675 62.97766866 -12.59930039 57.99522542 -12.6492157 53.01243591 C-12.70938501 46.99948509 -12.74014853 40.9870592 -12.74984932 34.97382164 C-12.7584117 32.68248073 -12.77724884 30.39115413 -12.80659294 28.09998512 C-12.84549318 24.88951624 -12.84654319 21.68109224 -12.83866882 18.47044373 C-12.85866943 17.52627594 -12.87867004 16.58210815 -12.89927673 15.60932922 C-12.83484466 9.74781802 -11.72647549 6.66605148 -7.62495518 2.42708111 C-5.05527972 0.23308596 -3.29174519 0.02991809 0 0 Z M1.50151062 18.94068909 C0.56440872 22.17521358 0.56353658 25.34117723 0.56782532 28.67967224 C0.56110809 29.43139709 0.55439087 30.18312195 0.54747009 30.95762634 C0.52849296 33.43942397 0.52456606 35.92095113 0.52241516 38.40281677 C0.51598698 40.13150646 0.5092586 41.86019505 0.50224304 43.58888245 C0.49027097 47.2119448 0.48656214 50.83491181 0.48774719 54.45799255 C0.48795037 59.09400724 0.46064985 63.72940881 0.42620277 68.36527824 C0.40393347 71.93585502 0.39990966 75.50628426 0.40096664 79.07692337 C0.39861067 80.78571307 0.3897873 82.49450684 0.37430191 84.203228 C0.35464087 86.59685083 0.36047948 88.98915179 0.37251282 91.38279724 C0.3612738 92.08429901 0.35003479 92.78580078 0.3384552 93.50856018 C0.40196991 98.49625643 1.66403777 101.3629638 4.85835266 105.16062927 C8.29964553 107.78373822 12.21729425 107.45650081 16.37739563 107.45115662 C17.12912048 107.45787384 17.88084534 107.46459106 18.65534973 107.47151184 C21.13714736 107.49048897 23.61867452 107.49441587 26.10054016 107.49656677 C27.82922984 107.50299496 29.55791844 107.50972333 31.28660583 107.51673889 C34.90966819 107.52871097 38.5326352 107.53241979 42.15571594 107.53123474 C46.79173063 107.53103157 51.4271322 107.55833209 56.06300163 107.59277916 C59.63357841 107.61504847 63.20400765 107.61907228 66.77464676 107.61801529 C68.48343646 107.62037126 70.19223023 107.62919464 71.90095139 107.64468002 C74.29457422 107.66434107 76.68687518 107.65850246 79.08052063 107.64646912 C79.7820224 107.65770813 80.48352417 107.66894714 81.20628357 107.68052673 C86.19397982 107.61701203 89.06068718 106.35494417 92.85835266 103.16062927 C95.48817303 99.71497264 95.14760513 95.79131326 95.13301086 91.62571716 C95.14067726 90.49572044 95.14067726 90.49572044 95.14849854 89.34289551 C95.16238195 86.8562777 95.16143875 84.36993745 95.15913391 81.88328552 C95.1630215 80.15220029 95.1673563 78.42111601 95.17211914 76.69003296 C95.17957863 73.06276863 95.17929202 69.43560163 95.17402649 65.80833435 C95.16823182 61.16335098 95.18516397 56.51876636 95.20837116 51.87384892 C95.22311305 48.29902828 95.22397959 44.72430358 95.22086143 41.14945793 C95.22137441 39.43706408 95.22660763 37.72466483 95.23667336 36.01230049 C95.24902962 33.61624976 95.24225144 31.22097451 95.23066711 28.82493591 C95.23831085 28.11983887 95.24595459 27.41474182 95.25382996 26.6882782 C95.20736448 22.33343222 94.61604615 19.56450247 91.85835266 16.16062927 C87.86951071 12.93549505 84.17588538 12.72483146 79.2257843 12.75657654 C78.47391342 12.74971329 77.72204254 12.74285004 76.9473877 12.73577881 C74.4709007 12.71753151 71.99504848 12.72134524 69.51850891 12.72703552 C67.7912475 12.72247263 66.06398792 12.71716674 64.33673096 12.71115112 C60.71959551 12.70222404 57.1026809 12.70463841 53.48554993 12.71458435 C48.85737598 12.72607218 44.22996522 12.7058637 39.60189152 12.67666721 C36.03558536 12.65829306 32.46946167 12.65860009 28.90312004 12.66430473 C27.19713203 12.66445781 25.49113154 12.65842495 23.78518867 12.64601326 C21.39641515 12.63108197 19.00904608 12.64179466 16.62031555 12.65892029 C15.92038483 12.64925232 15.2204541 12.63958435 14.49931335 12.62962341 C8.89328018 12.71349425 4.94042692 14.4291273 1.50151062 18.94068909 Z " fill="#00239C" transform="translate(12.141647338867188,-0.1606292724609375)"/>
+<path d="M0 0 C1.83749565 -0.01925285 1.83749565 -0.01925285 3.71211243 -0.03889465 C5.05571082 -0.04706237 6.39931378 -0.05450604 7.74291992 -0.0612793 C8.41886389 -0.06535482 9.09480785 -0.06943035 9.79123497 -0.07362938 C13.36960927 -0.09448933 16.94794524 -0.10879078 20.52636719 -0.11816406 C24.21867292 -0.12922282 27.91037503 -0.16361566 31.60246754 -0.20333576 C34.44468794 -0.22950944 37.28676476 -0.23785654 40.12909508 -0.24144554 C41.48984538 -0.2463104 42.8505893 -0.25792176 44.21122169 -0.27648354 C46.11853089 -0.30088366 48.02613384 -0.29748929 49.93359375 -0.29296875 C51.01788162 -0.29883499 52.10216949 -0.30470123 53.21931458 -0.31074524 C56.86654226 0.29400752 58.52626077 1.63652802 61.16088867 4.17700195 C62.54314883 6.94152228 62.25614254 9.22313826 62.22338867 12.30200195 C62.21436523 13.39770508 62.2053418 14.4934082 62.19604492 15.62231445 C62.18444336 16.46536133 62.1728418 17.3084082 62.16088867 18.17700195 C55.98619204 18.25123994 49.81175607 18.30564608 43.63671875 18.34179688 C41.53673781 18.35686967 39.43678828 18.37733137 37.33691406 18.40332031 C34.31592043 18.43976949 31.29534356 18.45674355 28.27416992 18.4699707 C27.33753006 18.48545456 26.4008902 18.50093842 25.43586731 18.51689148 C20.36464019 18.51819649 17.41383115 18.18564024 13.16088867 15.17700195 C9.52849054 15.00536525 9.52849054 15.00536525 6.16088867 16.17700195 C3.60317135 18.97450527 3.14615503 20.45505357 3.22338867 24.23950195 C4.31755912 27.66790268 5.21661132 29.1080503 8.16088867 31.17700195 C11.16088867 31.76033529 11.16088867 31.76033529 14.16088867 31.17700195 C14.70237549 30.68337158 15.2438623 30.18974121 15.80175781 29.68115234 C19.31463202 27.44139057 22.86528525 27.68961623 26.90698242 27.6730957 C27.75511002 27.6631105 28.60323761 27.65312531 29.47706604 27.64283752 C31.2681898 27.62767186 33.05939804 27.62061125 34.85058594 27.62109375 C37.58630291 27.61453095 40.31839175 27.56003661 43.0534668 27.50317383 C44.79629296 27.49437122 46.53913631 27.48841736 48.28198242 27.4855957 C49.09670517 27.46403915 49.91142792 27.4424826 50.75083923 27.42027283 C55.14475818 27.47311228 56.70438448 27.79056997 60.17651367 30.72973633 C62.43076923 34.64583974 62.87350852 37.623383 62.78979492 42.0246582 C62.79220688 42.75897568 62.79461884 43.49329315 62.79710388 44.24986267 C62.79612927 45.79374625 62.78282772 47.33766645 62.7578125 48.88134766 C62.72342562 51.23697056 62.73718793 53.588852 62.7565918 55.94458008 C62.74969426 57.45045532 62.7400227 58.95632095 62.72729492 60.4621582 C62.73473473 61.51077782 62.73473473 61.51077782 62.74232483 62.58058167 C62.62773275 67.363316 61.53998864 69.81290442 58.16088867 73.17700195 C55.46421666 74.52533796 53.34117034 74.32653048 50.32177734 74.35400391 C48.48428169 74.37325676 48.48428169 74.37325676 46.60966492 74.39289856 C45.26606653 74.40106628 43.92246357 74.40850995 42.57885742 74.4152832 C41.90291346 74.41935873 41.22696949 74.42343426 40.53054237 74.42763329 C36.95216808 74.44849324 33.37383211 74.46279468 29.79541016 74.47216797 C26.10310442 74.48322673 22.41140231 74.51761956 18.71930981 74.55733967 C15.8770894 74.58351334 13.03501259 74.59186045 10.19268227 74.59544945 C8.83193197 74.60031431 7.47118804 74.61192567 6.11055565 74.63048744 C4.20324645 74.65488757 2.2956435 74.6514932 0.38818359 74.64697266 C-0.69610428 74.6528389 -1.78039215 74.65870514 -2.89753723 74.66474915 C-6.54476492 74.05999638 -8.20448343 72.71747588 -10.83911133 70.17700195 C-12.22137149 67.41248163 -11.9343652 65.13086565 -11.90161133 62.05200195 C-11.89258789 60.95629883 -11.88356445 59.8605957 -11.87426758 58.73168945 C-11.86266602 57.88864258 -11.85106445 57.0455957 -11.83911133 56.17700195 C-5.66441469 56.10276397 0.51002128 56.04835782 6.68505859 56.01220703 C8.78503953 55.99713423 10.88498907 55.97667254 12.98486328 55.95068359 C16.00585691 55.91423442 19.02643378 55.89726036 22.04760742 55.8840332 C22.98424728 55.86854935 23.92088715 55.85306549 24.88591003 55.83711243 C29.95713715 55.83580742 32.9079462 56.16836367 37.16088867 59.17700195 C40.79328681 59.34863865 40.79328681 59.34863865 44.16088867 58.17700195 C46.71860599 55.37949863 47.17562231 53.89895034 47.09838867 50.11450195 C46.00421823 46.68610122 45.10516602 45.2459536 42.16088867 43.17700195 C39.16088867 42.59366862 39.16088867 42.59366862 36.16088867 43.17700195 C35.61940186 43.67063232 35.07791504 44.1642627 34.52001953 44.67285156 C31.00714532 46.91261334 27.4564921 46.66438768 23.41479492 46.6809082 C22.56666733 46.6908934 21.71853973 46.7008786 20.8447113 46.71116638 C19.05358754 46.72633205 17.2623793 46.73339265 15.47119141 46.73291016 C12.73547443 46.73947296 10.00338559 46.7939673 7.26831055 46.85083008 C5.52548439 46.85963269 3.78264103 46.86558654 2.03979492 46.8684082 C1.22507217 46.88996475 0.41034943 46.9115213 -0.42906189 46.93373108 C-4.82298084 46.88089163 -6.38260714 46.56343393 -9.85473633 43.62426758 C-12.10899189 39.70816417 -12.55173118 36.7306209 -12.46801758 32.3293457 C-12.47042953 31.59502823 -12.47284149 30.86071075 -12.47532654 30.10414124 C-12.47435192 28.56025766 -12.46105038 27.01633746 -12.43603516 25.47265625 C-12.40164828 23.11703335 -12.41541058 20.76515191 -12.43481445 18.40942383 C-12.42791692 16.90354858 -12.41824535 15.39768295 -12.40551758 13.8918457 C-12.41047745 13.19276596 -12.41543732 12.49368622 -12.42054749 11.77342224 C-12.30595541 6.9906879 -11.21821129 4.54109949 -7.83911133 1.17700195 C-5.14243932 -0.17133405 -3.019393 0.02747343 0 0 Z " fill="#00249C" transform="translate(34.839111328125,22.822998046875)"/>
+>>>>>>> 70e1bd5d (feat(CE): AIS Data store destination connector (#573))
 </svg>
diff --git a/integrations/lib/multiwoven/integrations/rollout.rb b/integrations/lib/multiwoven/integrations/rollout.rb
index 4abff0f7..6d9488b8 100644
--- a/integrations/lib/multiwoven/integrations/rollout.rb
+++ b/integrations/lib/multiwoven/integrations/rollout.rb
@@ -2,7 +2,7 @@
 
 module Multiwoven
   module Integrations
-    VERSION = "0.14.2"
+    VERSION = "0.15.0"
 
     ENABLED_SOURCES = %w[
       Snowflake
@@ -43,6 +43,7 @@ module Integrations
       MicrosoftExcel
       MicrosoftSql
       Mailchimp
+      AISDataStore
     ].freeze
   end
 end
diff --git a/integrations/spec/multiwoven/integrations/destination/ais_data_store/client_spec.rb b/integrations/spec/multiwoven/integrations/destination/ais_data_store/client_spec.rb
new file mode 100644
index 00000000..8419c92f
--- /dev/null
+++ b/integrations/spec/multiwoven/integrations/destination/ais_data_store/client_spec.rb
@@ -0,0 +1,215 @@
+# frozen_string_literal: true
+
+RSpec.describe Multiwoven::Integrations::Destination::AISDataStore::Client do
+  let(:client) { Multiwoven::Integrations::Destination::AISDataStore::Client.new }
+  let(:sync_config) do
+    {
+      "source": {
+        "name": "PostgresqlSourceConnector",
+        "type": "source",
+        "connection_specification": {
+          "credentials": {
+            "auth_type": "username/password",
+            "username": ENV["POSTGRESQL_USERNAME"],
+            "password": ENV["POSTGRESQL_PASSWORD"]
+          },
+          "host": "test.pg.com",
+          "port": "8080",
+          "database": "test_database",
+          "schema": "test_schema"
+        }
+      },
+      "destination": {
+        "name": "AISDataStore",
+        "type": "destination",
+        "connection_specification": {
+          "credentials": {
+            "auth_type": "username/password",
+            "username": ENV["POSTGRESQL_USERNAME"],
+            "password": ENV["POSTGRESQL_PASSWORD"]
+          },
+          "host": "test.pg.com",
+          "port": "8080",
+          "database": "test_database",
+          "schema": "test_schema"
+        }
+      },
+      "model": {
+        "name": "ExamplePostgresqlModel",
+        "query": "SELECT * FROM contacts;",
+        "query_type": "raw_sql",
+        "primary_key": "id"
+      },
+      "stream": {
+        "name": "users", "action": "create",
+        "json_schema": { "user_id": "string", "email": "string", "location": "string" },
+        "supported_sync_modes": %w[full_refresh incremental]
+      },
+      "sync_mode": "full_refresh",
+      "cursor_field": "timestamp",
+      "destination_sync_mode": "upsert",
+      "sync_id": "1"
+    }
+  end
+
+  let(:pg_connection) { instance_double(PG::Connection) }
+  let(:pg_result) { instance_double(PG::Result) }
+
+  let(:records) do
+    [
+      Multiwoven::Integrations::Protocol::RecordMessage.new(
+        data: {
+          email: "user1@example.com",
+          location: "New York",
+          user_id: 1
+        },
+        emitted_at: Time.now.to_i
+      ),
+      Multiwoven::Integrations::Protocol::RecordMessage.new(
+        data: {
+          email: "user2@example.com",
+          location: "San Francisco",
+          user_id: 2
+        },
+        emitted_at: Time.now.to_i
+      )
+    ]
+  end
+
+  describe "#check_connection" do
+    context "when the connection is successful" do
+      it "returns a succeeded connection status" do
+        allow(Sequel).to receive(:postgres).and_return(true)
+        allow(PG).to receive(:connect).and_return(pg_connection)
+        message = client.check_connection(sync_config[:source][:connection_specification])
+        result = message.connection_status
+
+        expect(result.status).to eq("succeeded")
+        expect(result.message).to be_nil
+      end
+    end
+
+    context "when the connection fails" do
+      it "returns a failed connection status with an error message" do
+        allow(PG).to receive(:connect).and_raise(PG::Error.new("Connection failed"))
+
+        message = client.check_connection(sync_config[:source][:connection_specification])
+        result = message.connection_status
+        expect(result.status).to eq("failed")
+        expect(result.message).to include("Connection failed")
+      end
+    end
+  end
+
+  # write specs
+
+  describe "#write" do
+    context "success" do
+      it "write records successfully" do
+        s_config = Multiwoven::Integrations::Protocol::SyncConfig.from_json(sync_config.to_json)
+        s_config.sync_run_id = "33"
+        allow(PG).to receive(:connect).and_return(pg_connection)
+
+        allow(pg_connection).to receive(:exec).and_return(true)
+
+        tracking = subject.write(s_config, [records.first.data.transform_keys(&:to_s)]).tracking
+        expect(tracking.success).to eql(1)
+        log_message = tracking.logs.first
+        expect(log_message).to be_a(Multiwoven::Integrations::Protocol::LogMessage)
+        expect(log_message.level).to eql("info")
+
+        expect(log_message.message).to include("request")
+        expect(log_message.message).to include("response")
+      end
+
+      it "write records successfully on update record action destination_update" do
+        s_config = Multiwoven::Integrations::Protocol::SyncConfig.from_json(sync_config.to_json)
+        s_config.sync_run_id = "33"
+        allow(PG).to receive(:connect).and_return(pg_connection)
+
+        allow(pg_connection).to receive(:exec).and_return(true)
+
+        tracking = subject.write(s_config, [records.first.data.transform_keys(&:to_s)], "destination_update").tracking
+        expect(tracking.success).to eql(1)
+        expect(tracking.logs.count).to eql(1)
+        log_message = tracking.logs.first
+        expect(log_message).to be_a(Multiwoven::Integrations::Protocol::LogMessage)
+        expect(log_message.level).to eql("info")
+
+        expect(log_message.message).to include("request")
+        expect(log_message.message).to include("response")
+      end
+    end
+
+    context "failure" do
+      it "handle record write failures" do
+        s_config = Multiwoven::Integrations::Protocol::SyncConfig.from_json(sync_config.to_json)
+        s_config.sync_run_id = "34"
+
+        allow(PG).to receive(:connect).and_return(pg_connection)
+
+        allow(pg_connection).to receive(:exec).and_raise(StandardError.new("test error"))
+
+        tracking = subject.write(s_config, [records.first.data.transform_keys(&:to_s)]).tracking
+        expect(tracking.failed).to eql(1)
+        expect(tracking.logs.count).to eql(1)
+        log_message = tracking.logs.first
+        expect(log_message).to be_a(Multiwoven::Integrations::Protocol::LogMessage)
+        expect(log_message.level).to eql("error")
+        expect(log_message.message).to include("request")
+        expect(log_message.message).to include("\"response\":\"test error\"")
+      end
+    end
+  end
+
+  describe "#discover" do
+    it "discovers schema successfully" do
+      allow(PG).to receive(:connect).and_return(pg_connection)
+      discovery_query = "SELECT table_name, column_name, data_type, is_nullable\n" \
+                      "                 FROM information_schema.columns\n" \
+                      "                 WHERE table_schema = 'test_schema' AND table_catalog = 'test_database'\n" \
+                      "                 ORDER BY table_name, ordinal_position;"
+      allow(pg_connection).to receive(:exec).with(discovery_query).and_return(
+        [
+          {
+            "table_name" => "combined_users", "column_name" => "city", "data_type" => "varchar", "is_nullable" => "YES"
+          }
+        ]
+      )
+      allow(pg_connection).to receive(:close).and_return(true)
+      message = client.discover(sync_config[:source][:connection_specification])
+
+      expect(message.catalog).to be_an(Multiwoven::Integrations::Protocol::Catalog)
+      first_stream = message.catalog.streams.first
+      expect(first_stream).to be_a(Multiwoven::Integrations::Protocol::Stream)
+      expect(first_stream.name).to eq("combined_users")
+      expect(first_stream.json_schema).to be_an(Hash)
+      expect(first_stream.json_schema["type"]).to eq("object")
+      expect(first_stream.json_schema["properties"]).to eq({ "city" => { "type" => %w[string null] } })
+    end
+
+    it "discover schema failure" do
+      allow(client).to receive(:create_connection).and_raise(StandardError.new("test error"))
+      expect(client).to receive(:handle_exception).with(
+        an_instance_of(StandardError), {
+          context: "POSTGRESQL:DISCOVER:EXCEPTION",
+          type: "error"
+        }
+      )
+      client.discover(sync_config[:source][:connection_specification])
+    end
+  end
+
+  describe "#meta_data" do
+    it "client class_name and meta name is same" do
+      meta_name = client.class.to_s.split("::")[-2]
+      expect(client.send(:meta_data)[:data][:name]).to eq(meta_name)
+    end
+  end
+
+  describe "method definition" do
+    it "defines a private #query method" do
+      expect(described_class.private_instance_methods).to include(:query)
+    end
+  end
+end

From 8d57a2442671b8dafea65d2c7cb0c6593b484ea7 Mon Sep 17 00:00:00 2001
From: afthab vp <afthab@multiwoven.com>
Date: Wed, 13 Nov 2024 15:24:40 +0530
Subject: [PATCH 33/48] fix(CE): ais data store icon (#467)

---
 .../integrations/destination/ais_data_store/icon.svg       | 7 -------
 1 file changed, 7 deletions(-)

diff --git a/integrations/lib/multiwoven/integrations/destination/ais_data_store/icon.svg b/integrations/lib/multiwoven/integrations/destination/ais_data_store/icon.svg
index eb2d575c..277ee307 100644
--- a/integrations/lib/multiwoven/integrations/destination/ais_data_store/icon.svg
+++ b/integrations/lib/multiwoven/integrations/destination/ais_data_store/icon.svg
@@ -1,11 +1,4 @@
-<<<<<<< HEAD
 <svg width="66" height="64" viewBox="0 0 66 64" fill="none" xmlns="http://www.w3.org/2000/svg">
 <path d="M24.5072 18.9904C26.4821 18.9904 28.2101 20.0795 29.1976 21.6527H51.2916V18.2643C51.2916 15.3599 48.9464 13.0607 45.9841 13.0607H18.2123C15.4968 13.0607 13.3985 15.2389 13.3985 17.7802V33.1491C13.3985 34.9643 14.8796 36.4165 16.7311 36.4165H34.9988C35.8628 34.4802 37.8377 33.1491 40.0595 33.1491C43.1452 33.1491 45.7373 35.6904 45.7373 38.8368C45.7373 41.9831 43.1452 44.5244 40.0595 44.5244C38.4549 44.5244 36.9737 43.7984 35.8628 42.5882H13.1516V46.5817C13.1516 49.002 15.1265 50.9382 17.5951 50.9382H47.8356C49.687 50.9382 51.1682 49.486 51.0448 47.6708V32.181C51.0448 30.0027 49.1933 28.1875 46.9716 28.1875H28.9507C27.9633 29.5186 26.3587 30.3657 24.5072 30.3657C21.4215 30.3657 18.8294 27.8244 18.8294 24.6781C18.8294 21.5317 21.4215 18.9904 24.5072 18.9904Z" fill="#00249C"/>
 <path d="M57.0292 0H8.02359C3.57976 0 0 3.54198 0 7.93893V56.0611C0 60.458 3.57976 64 8.02359 64H57.0292C61.4731 64 65.0528 60.458 65.0528 56.0611V7.93893C65.1763 3.54198 61.4731 0 57.0292 0ZM59.1277 51.6641C59.1277 55.2061 56.1652 58.1374 52.5854 58.1374H12.4674C8.88767 58.1374 5.92512 55.2061 5.92512 51.6641V12.3359C5.92512 8.79389 8.88767 5.8626 12.4674 5.8626H52.7088C56.2886 5.8626 59.2512 8.79389 59.2512 12.3359V51.6641H59.1277Z" fill="#00249C"/>
-=======
-<?xml version="1.0" encoding="UTF-8"?>
-<svg version="1.1" xmlns="http://www.w3.org/2000/svg" width="120" height="120">
-<path d="M0 0 C0.86770523 -0.01004562 1.73541046 -0.02009125 2.62940979 -0.03044128 C3.58369873 -0.03627228 4.53798767 -0.04210327 5.52119446 -0.04811096 C6.5246138 -0.05789474 7.52803314 -0.06767853 8.56185913 -0.07775879 C11.89374965 -0.10775492 15.22559986 -0.12911119 18.55757141 -0.14796448 C20.26279433 -0.15782222 20.26279433 -0.15782222 22.0024662 -0.1678791 C28.02106609 -0.20049291 34.03964812 -0.22493756 40.05830383 -0.24440002 C45.04131327 -0.26221014 50.02375652 -0.29702378 55.00654602 -0.34693909 C61.01949684 -0.4071084 67.03192274 -0.43787191 73.04516029 -0.44757271 C75.3365012 -0.45613508 77.62782781 -0.47497223 79.91899681 -0.50431633 C83.12946569 -0.54321656 86.33788969 -0.54426658 89.54853821 -0.53639221 C90.96478989 -0.56639313 90.96478989 -0.56639313 92.40965271 -0.59700012 C98.27116391 -0.53256805 101.35293046 0.57580113 105.59190083 4.67732143 C107.78589597 7.24699689 107.98906384 9.01053142 108.01898193 12.30227661 C108.02902756 13.16998184 108.03907318 14.03768707 108.04942322 14.9316864 C108.05525421 15.88597534 108.06108521 16.84026428 108.0670929 17.82347107 C108.07687668 18.82689041 108.08666046 19.83030975 108.09674072 20.86413574 C108.12673686 24.19602626 108.14809313 27.52787647 108.16694641 30.85984802 C108.17351824 31.9966633 108.18009007 33.13347858 108.18686104 34.30474281 C108.21947484 40.3233427 108.24391949 46.34192473 108.26338196 52.36058044 C108.28119207 57.34358989 108.31600571 62.32603313 108.36592102 67.30882263 C108.42609034 73.32177345 108.45685385 79.33419935 108.46655464 85.3474369 C108.47511702 87.63877781 108.49395416 89.93010442 108.52329826 92.22127342 C108.5621985 95.43174231 108.56324851 98.6401663 108.55537415 101.85081482 C108.57537476 102.7949826 108.59537537 103.73915039 108.61598206 104.71192932 C108.55154998 110.57344052 107.44318081 113.65520707 103.3416605 117.89417744 C100.77198505 120.08817258 99.00845051 120.29134045 95.71670532 120.32125854 C94.84900009 120.33130417 93.98129486 120.34134979 93.08729553 120.35169983 C91.65586212 120.36044632 91.65586212 120.36044632 90.19551086 120.36936951 C89.19209152 120.37915329 88.18867218 120.38893707 87.15484619 120.39901733 C83.82295567 120.42901347 80.49110546 120.45036974 77.15913391 120.46922302 C75.45391099 120.47908076 75.45391099 120.47908076 73.71423912 120.48913765 C67.69563923 120.52175145 61.6770572 120.5461961 55.65840149 120.56565857 C50.67539205 120.58346868 45.69294881 120.61828232 40.7101593 120.66819763 C34.69720848 120.72836695 28.68478258 120.75913046 22.67154503 120.76883125 C20.38020412 120.77739363 18.08887751 120.79623078 15.79770851 120.82557487 C12.58723963 120.86447511 9.37881563 120.86552512 6.16816711 120.85765076 C4.75191544 120.88765167 4.75191544 120.88765167 3.30705261 120.91825867 C-2.55445859 120.85382659 -5.63622513 119.74545742 -9.8751955 115.64393711 C-12.06919065 113.07426166 -12.27235852 111.31072713 -12.30227661 108.01898193 C-12.31232224 107.1512767 -12.32236786 106.28357147 -12.3327179 105.38957214 C-12.34146439 103.95813873 -12.34146439 103.95813873 -12.35038757 102.49778748 C-12.36017136 101.49436813 -12.36995514 100.49094879 -12.3800354 99.4571228 C-12.41003153 96.12523229 -12.4313878 92.79338207 -12.45024109 89.46141052 C-12.45681292 88.32459524 -12.46338474 87.18777996 -12.47015572 86.01651573 C-12.50276952 79.99791585 -12.52721417 73.97933381 -12.54667664 67.9606781 C-12.56448675 62.97766866 -12.59930039 57.99522542 -12.6492157 53.01243591 C-12.70938501 46.99948509 -12.74014853 40.9870592 -12.74984932 34.97382164 C-12.7584117 32.68248073 -12.77724884 30.39115413 -12.80659294 28.09998512 C-12.84549318 24.88951624 -12.84654319 21.68109224 -12.83866882 18.47044373 C-12.85866943 17.52627594 -12.87867004 16.58210815 -12.89927673 15.60932922 C-12.83484466 9.74781802 -11.72647549 6.66605148 -7.62495518 2.42708111 C-5.05527972 0.23308596 -3.29174519 0.02991809 0 0 Z M1.50151062 18.94068909 C0.56440872 22.17521358 0.56353658 25.34117723 0.56782532 28.67967224 C0.56110809 29.43139709 0.55439087 30.18312195 0.54747009 30.95762634 C0.52849296 33.43942397 0.52456606 35.92095113 0.52241516 38.40281677 C0.51598698 40.13150646 0.5092586 41.86019505 0.50224304 43.58888245 C0.49027097 47.2119448 0.48656214 50.83491181 0.48774719 54.45799255 C0.48795037 59.09400724 0.46064985 63.72940881 0.42620277 68.36527824 C0.40393347 71.93585502 0.39990966 75.50628426 0.40096664 79.07692337 C0.39861067 80.78571307 0.3897873 82.49450684 0.37430191 84.203228 C0.35464087 86.59685083 0.36047948 88.98915179 0.37251282 91.38279724 C0.3612738 92.08429901 0.35003479 92.78580078 0.3384552 93.50856018 C0.40196991 98.49625643 1.66403777 101.3629638 4.85835266 105.16062927 C8.29964553 107.78373822 12.21729425 107.45650081 16.37739563 107.45115662 C17.12912048 107.45787384 17.88084534 107.46459106 18.65534973 107.47151184 C21.13714736 107.49048897 23.61867452 107.49441587 26.10054016 107.49656677 C27.82922984 107.50299496 29.55791844 107.50972333 31.28660583 107.51673889 C34.90966819 107.52871097 38.5326352 107.53241979 42.15571594 107.53123474 C46.79173063 107.53103157 51.4271322 107.55833209 56.06300163 107.59277916 C59.63357841 107.61504847 63.20400765 107.61907228 66.77464676 107.61801529 C68.48343646 107.62037126 70.19223023 107.62919464 71.90095139 107.64468002 C74.29457422 107.66434107 76.68687518 107.65850246 79.08052063 107.64646912 C79.7820224 107.65770813 80.48352417 107.66894714 81.20628357 107.68052673 C86.19397982 107.61701203 89.06068718 106.35494417 92.85835266 103.16062927 C95.48817303 99.71497264 95.14760513 95.79131326 95.13301086 91.62571716 C95.14067726 90.49572044 95.14067726 90.49572044 95.14849854 89.34289551 C95.16238195 86.8562777 95.16143875 84.36993745 95.15913391 81.88328552 C95.1630215 80.15220029 95.1673563 78.42111601 95.17211914 76.69003296 C95.17957863 73.06276863 95.17929202 69.43560163 95.17402649 65.80833435 C95.16823182 61.16335098 95.18516397 56.51876636 95.20837116 51.87384892 C95.22311305 48.29902828 95.22397959 44.72430358 95.22086143 41.14945793 C95.22137441 39.43706408 95.22660763 37.72466483 95.23667336 36.01230049 C95.24902962 33.61624976 95.24225144 31.22097451 95.23066711 28.82493591 C95.23831085 28.11983887 95.24595459 27.41474182 95.25382996 26.6882782 C95.20736448 22.33343222 94.61604615 19.56450247 91.85835266 16.16062927 C87.86951071 12.93549505 84.17588538 12.72483146 79.2257843 12.75657654 C78.47391342 12.74971329 77.72204254 12.74285004 76.9473877 12.73577881 C74.4709007 12.71753151 71.99504848 12.72134524 69.51850891 12.72703552 C67.7912475 12.72247263 66.06398792 12.71716674 64.33673096 12.71115112 C60.71959551 12.70222404 57.1026809 12.70463841 53.48554993 12.71458435 C48.85737598 12.72607218 44.22996522 12.7058637 39.60189152 12.67666721 C36.03558536 12.65829306 32.46946167 12.65860009 28.90312004 12.66430473 C27.19713203 12.66445781 25.49113154 12.65842495 23.78518867 12.64601326 C21.39641515 12.63108197 19.00904608 12.64179466 16.62031555 12.65892029 C15.92038483 12.64925232 15.2204541 12.63958435 14.49931335 12.62962341 C8.89328018 12.71349425 4.94042692 14.4291273 1.50151062 18.94068909 Z " fill="#00239C" transform="translate(12.141647338867188,-0.1606292724609375)"/>
-<path d="M0 0 C1.83749565 -0.01925285 1.83749565 -0.01925285 3.71211243 -0.03889465 C5.05571082 -0.04706237 6.39931378 -0.05450604 7.74291992 -0.0612793 C8.41886389 -0.06535482 9.09480785 -0.06943035 9.79123497 -0.07362938 C13.36960927 -0.09448933 16.94794524 -0.10879078 20.52636719 -0.11816406 C24.21867292 -0.12922282 27.91037503 -0.16361566 31.60246754 -0.20333576 C34.44468794 -0.22950944 37.28676476 -0.23785654 40.12909508 -0.24144554 C41.48984538 -0.2463104 42.8505893 -0.25792176 44.21122169 -0.27648354 C46.11853089 -0.30088366 48.02613384 -0.29748929 49.93359375 -0.29296875 C51.01788162 -0.29883499 52.10216949 -0.30470123 53.21931458 -0.31074524 C56.86654226 0.29400752 58.52626077 1.63652802 61.16088867 4.17700195 C62.54314883 6.94152228 62.25614254 9.22313826 62.22338867 12.30200195 C62.21436523 13.39770508 62.2053418 14.4934082 62.19604492 15.62231445 C62.18444336 16.46536133 62.1728418 17.3084082 62.16088867 18.17700195 C55.98619204 18.25123994 49.81175607 18.30564608 43.63671875 18.34179688 C41.53673781 18.35686967 39.43678828 18.37733137 37.33691406 18.40332031 C34.31592043 18.43976949 31.29534356 18.45674355 28.27416992 18.4699707 C27.33753006 18.48545456 26.4008902 18.50093842 25.43586731 18.51689148 C20.36464019 18.51819649 17.41383115 18.18564024 13.16088867 15.17700195 C9.52849054 15.00536525 9.52849054 15.00536525 6.16088867 16.17700195 C3.60317135 18.97450527 3.14615503 20.45505357 3.22338867 24.23950195 C4.31755912 27.66790268 5.21661132 29.1080503 8.16088867 31.17700195 C11.16088867 31.76033529 11.16088867 31.76033529 14.16088867 31.17700195 C14.70237549 30.68337158 15.2438623 30.18974121 15.80175781 29.68115234 C19.31463202 27.44139057 22.86528525 27.68961623 26.90698242 27.6730957 C27.75511002 27.6631105 28.60323761 27.65312531 29.47706604 27.64283752 C31.2681898 27.62767186 33.05939804 27.62061125 34.85058594 27.62109375 C37.58630291 27.61453095 40.31839175 27.56003661 43.0534668 27.50317383 C44.79629296 27.49437122 46.53913631 27.48841736 48.28198242 27.4855957 C49.09670517 27.46403915 49.91142792 27.4424826 50.75083923 27.42027283 C55.14475818 27.47311228 56.70438448 27.79056997 60.17651367 30.72973633 C62.43076923 34.64583974 62.87350852 37.623383 62.78979492 42.0246582 C62.79220688 42.75897568 62.79461884 43.49329315 62.79710388 44.24986267 C62.79612927 45.79374625 62.78282772 47.33766645 62.7578125 48.88134766 C62.72342562 51.23697056 62.73718793 53.588852 62.7565918 55.94458008 C62.74969426 57.45045532 62.7400227 58.95632095 62.72729492 60.4621582 C62.73473473 61.51077782 62.73473473 61.51077782 62.74232483 62.58058167 C62.62773275 67.363316 61.53998864 69.81290442 58.16088867 73.17700195 C55.46421666 74.52533796 53.34117034 74.32653048 50.32177734 74.35400391 C48.48428169 74.37325676 48.48428169 74.37325676 46.60966492 74.39289856 C45.26606653 74.40106628 43.92246357 74.40850995 42.57885742 74.4152832 C41.90291346 74.41935873 41.22696949 74.42343426 40.53054237 74.42763329 C36.95216808 74.44849324 33.37383211 74.46279468 29.79541016 74.47216797 C26.10310442 74.48322673 22.41140231 74.51761956 18.71930981 74.55733967 C15.8770894 74.58351334 13.03501259 74.59186045 10.19268227 74.59544945 C8.83193197 74.60031431 7.47118804 74.61192567 6.11055565 74.63048744 C4.20324645 74.65488757 2.2956435 74.6514932 0.38818359 74.64697266 C-0.69610428 74.6528389 -1.78039215 74.65870514 -2.89753723 74.66474915 C-6.54476492 74.05999638 -8.20448343 72.71747588 -10.83911133 70.17700195 C-12.22137149 67.41248163 -11.9343652 65.13086565 -11.90161133 62.05200195 C-11.89258789 60.95629883 -11.88356445 59.8605957 -11.87426758 58.73168945 C-11.86266602 57.88864258 -11.85106445 57.0455957 -11.83911133 56.17700195 C-5.66441469 56.10276397 0.51002128 56.04835782 6.68505859 56.01220703 C8.78503953 55.99713423 10.88498907 55.97667254 12.98486328 55.95068359 C16.00585691 55.91423442 19.02643378 55.89726036 22.04760742 55.8840332 C22.98424728 55.86854935 23.92088715 55.85306549 24.88591003 55.83711243 C29.95713715 55.83580742 32.9079462 56.16836367 37.16088867 59.17700195 C40.79328681 59.34863865 40.79328681 59.34863865 44.16088867 58.17700195 C46.71860599 55.37949863 47.17562231 53.89895034 47.09838867 50.11450195 C46.00421823 46.68610122 45.10516602 45.2459536 42.16088867 43.17700195 C39.16088867 42.59366862 39.16088867 42.59366862 36.16088867 43.17700195 C35.61940186 43.67063232 35.07791504 44.1642627 34.52001953 44.67285156 C31.00714532 46.91261334 27.4564921 46.66438768 23.41479492 46.6809082 C22.56666733 46.6908934 21.71853973 46.7008786 20.8447113 46.71116638 C19.05358754 46.72633205 17.2623793 46.73339265 15.47119141 46.73291016 C12.73547443 46.73947296 10.00338559 46.7939673 7.26831055 46.85083008 C5.52548439 46.85963269 3.78264103 46.86558654 2.03979492 46.8684082 C1.22507217 46.88996475 0.41034943 46.9115213 -0.42906189 46.93373108 C-4.82298084 46.88089163 -6.38260714 46.56343393 -9.85473633 43.62426758 C-12.10899189 39.70816417 -12.55173118 36.7306209 -12.46801758 32.3293457 C-12.47042953 31.59502823 -12.47284149 30.86071075 -12.47532654 30.10414124 C-12.47435192 28.56025766 -12.46105038 27.01633746 -12.43603516 25.47265625 C-12.40164828 23.11703335 -12.41541058 20.76515191 -12.43481445 18.40942383 C-12.42791692 16.90354858 -12.41824535 15.39768295 -12.40551758 13.8918457 C-12.41047745 13.19276596 -12.41543732 12.49368622 -12.42054749 11.77342224 C-12.30595541 6.9906879 -11.21821129 4.54109949 -7.83911133 1.17700195 C-5.14243932 -0.17133405 -3.019393 0.02747343 0 0 Z " fill="#00249C" transform="translate(34.839111328125,22.822998046875)"/>
->>>>>>> 70e1bd5d (feat(CE): AIS Data store destination connector (#573))
 </svg>

From ad0848067a6be330a6e7f41d41be4a1e1714f57d Mon Sep 17 00:00:00 2001
From: "github-actions[bot]"
 <41898282+github-actions[bot]@users.noreply.github.com>
Date: Thu, 14 Nov 2024 13:15:35 +0530
Subject: [PATCH 34/48] chore(CE): update integration gem version (#457)

Co-authored-by: afthab vp <afthab@multiwoven.com>
---
 .github/workflows/server-ci.yml | 20 ++++++--
 server/Gemfile                  |  2 +-
 server/Gemfile.lock             | 87 ++++++++++++++++++---------------
 3 files changed, 63 insertions(+), 46 deletions(-)

diff --git a/.github/workflows/server-ci.yml b/.github/workflows/server-ci.yml
index 347c38b8..3a79bc4c 100644
--- a/.github/workflows/server-ci.yml
+++ b/.github/workflows/server-ci.yml
@@ -44,11 +44,11 @@ jobs:
 
       - name: Download and Extract DuckDB ODBC driver
         run: |
-            wget -O duckdb.zip https://github.com/duckdb/duckdb/releases/download/v1.0.0/libduckdb-linux-amd64.zip
-            unzip duckdb.zip -d libduckdb
-            sudo mv libduckdb/duckdb.* /usr/local/include
-            sudo mv libduckdb/libduckdb.so /usr/local/lib
-            sudo ldconfig /usr/local/lib
+          wget -O duckdb.zip https://github.com/duckdb/duckdb/releases/download/v1.0.0/libduckdb-linux-amd64.zip
+          unzip duckdb.zip -d libduckdb
+          sudo mv libduckdb/duckdb.* /usr/local/include
+          sudo mv libduckdb/libduckdb.so /usr/local/lib
+          sudo ldconfig /usr/local/lib
 
       - name: Download and Install Oracle Instant Client
         run: |
@@ -58,6 +58,16 @@ jobs:
           sudo alien -i --scripts oracle-instantclient*.rpm
           rm -f oracle-instantclient*.rpm
 
+      - name: Install FreeTDS for TinyTDS gem
+        run: |
+          sudo apt-get install libc6-dev
+          wget http://www.freetds.org/files/stable/freetds-1.4.10.tar.gz
+          tar -xzf freetds-1.4.10.tar.gz
+          cd freetds-1.4.10
+          ./configure --prefix=/usr/local --with-tdsver=7.4
+          sudo make
+          sudo make install
+
       - name: Bundle Install
         run: bundle install
         working-directory: ./server
diff --git a/server/Gemfile b/server/Gemfile
index 729c1452..37a78cae 100644
--- a/server/Gemfile
+++ b/server/Gemfile
@@ -13,7 +13,7 @@ gem "interactor", "~> 3.0"
 
 gem "ruby-odbc", git: "https://github.com/Multiwoven/ruby-odbc.git"
 
-gem "multiwoven-integrations", "~> 0.11.6"
+gem "multiwoven-integrations", "~> 0.15.0"
 
 gem "temporal-ruby", github: "coinbase/temporal-ruby"
 
diff --git a/server/Gemfile.lock b/server/Gemfile.lock
index 4eea7cb2..f2866920 100644
--- a/server/Gemfile.lock
+++ b/server/Gemfile.lock
@@ -16,6 +16,9 @@ GIT
 GEM
   remote: https://rubygems.org/
   specs:
+    MailchimpMarketing (3.0.80)
+      excon (>= 0.76.0, < 1)
+      json (~> 2.1, >= 2.1.0)
     aasm (5.5.0)
       concurrent-ruby (~> 1.0)
     actioncable (7.1.2)
@@ -108,22 +111,22 @@ GEM
     appsignal (3.7.5)
       rack
     ast (2.4.2)
-    async (2.17.0)
-      console (~> 1.26)
+    async (2.20.0)
+      console (~> 1.29)
       fiber-annotation
       io-event (~> 1.6, >= 1.6.5)
-    async-http (0.80.0)
+    async-http (0.83.1)
       async (>= 2.10.2)
-      async-pool (~> 0.7)
-      io-endpoint (~> 0.11)
-      io-stream (~> 0.4)
-      protocol-http (~> 0.37)
-      protocol-http1 (~> 0.27)
+      async-pool (~> 0.9)
+      io-endpoint (~> 0.14)
+      io-stream (~> 0.6)
+      metrics (~> 0.12)
+      protocol-http (~> 0.43)
+      protocol-http1 (>= 0.28.1)
       protocol-http2 (~> 0.19)
-      traces (>= 0.10)
-    async-pool (0.8.1)
+      traces (~> 0.10)
+    async-pool (0.10.1)
       async (>= 1.25)
-      metrics
       traces
     async-websocket (0.30.0)
       async-http (~> 0.76)
@@ -1664,7 +1667,7 @@ GEM
       activesupport
     concurrent-ruby (1.2.3)
     connection_pool (2.4.1)
-    console (1.27.0)
+    console (1.29.0)
       fiber-annotation
       fiber-local (~> 1.1)
       json
@@ -1736,7 +1739,7 @@ GEM
       dry-initializer (~> 3.0)
       dry-schema (>= 1.12, < 2)
       zeitwerk (~> 2.6)
-    duckdb (1.1.0.1)
+    duckdb (1.1.3.0)
       bigdecimal (>= 3.1.4)
     ecma-re-validator (0.4.0)
       regexp_parser (~> 2.2)
@@ -1745,6 +1748,7 @@ GEM
       tzinfo
     ethon (0.16.0)
       ffi (>= 1.15.0)
+    excon (0.112.0)
     factory_bot (6.4.5)
       activesupport (>= 5.0.0)
     factory_bot_rails (6.4.3)
@@ -1785,15 +1789,15 @@ GEM
       googleapis-common-protos-types (>= 1.11.0, < 2.a)
       googleauth (~> 1.9)
       grpc (~> 1.59)
-    git (2.3.0)
+    git (2.3.1)
       activesupport (>= 5.0)
       addressable (~> 2.8)
       process_executer (~> 1.1)
       rchardet (~> 1.8)
-    gli (2.21.5)
+    gli (2.22.0)
     globalid (1.2.1)
       activesupport (>= 6.1)
-    google-apis-bigquery_v2 (0.78.0)
+    google-apis-bigquery_v2 (0.80.0)
       google-apis-core (>= 0.15.0, < 2.a)
     google-apis-core (0.15.1)
       addressable (~> 2.5, >= 2.5.1)
@@ -1803,9 +1807,9 @@ GEM
       mutex_m
       representable (~> 3.0)
       retriable (>= 2.0, < 4.a)
-    google-apis-sheets_v4 (0.36.0)
+    google-apis-sheets_v4 (0.38.0)
       google-apis-core (>= 0.15.0, < 2.a)
-    google-cloud-ai_platform-v1 (0.53.0)
+    google-cloud-ai_platform-v1 (0.55.0)
       gapic-common (>= 0.21.1, < 2.a)
       google-cloud-errors (~> 1.0)
       google-cloud-location (>= 0.7, < 2.a)
@@ -1821,7 +1825,7 @@ GEM
     google-cloud-core (1.7.1)
       google-cloud-env (>= 1.0, < 3.a)
       google-cloud-errors (~> 1.0)
-    google-cloud-env (2.2.0)
+    google-cloud-env (2.2.1)
       faraday (>= 1.0, < 3.a)
     google-cloud-errors (1.4.0)
     google-cloud-location (0.8.1)
@@ -1841,7 +1845,7 @@ GEM
       grpc (~> 1.41)
     googleapis-common-protos-types (1.11.0)
       google-protobuf (~> 3.18)
-    googleauth (1.11.0)
+    googleauth (1.11.2)
       faraday (>= 1.0, < 3.a)
       google-cloud-env (~> 2.1)
       jwt (>= 1.4, < 3.0)
@@ -1870,7 +1874,7 @@ GEM
     hashie (5.0.0)
     htmlentities (4.3.4)
     httpclient (2.8.3)
-    hubspot-api-client (18.0.0)
+    hubspot-api-client (19.0.0)
       json (~> 2.1, >= 2.1.0)
       typhoeus (~> 1.4.0)
     i18n (1.14.1)
@@ -1879,9 +1883,9 @@ GEM
     inflection (1.0.0)
     interactor (3.1.2)
     io-console (0.7.1)
-    io-endpoint (0.13.1)
-    io-event (1.6.5)
-    io-stream (0.4.3)
+    io-endpoint (0.14.0)
+    io-event (1.7.3)
+    io-stream (0.6.1)
     irb (1.11.1)
       rdoc
       reline (>= 0.4.2)
@@ -1925,13 +1929,14 @@ GEM
       net-pop
       net-smtp
     marcel (1.0.2)
-    metrics (0.10.2)
+    metrics (0.12.1)
     mini_mime (1.1.5)
     minitest (5.21.2)
     msgpack (1.7.2)
     multi_json (1.15.0)
     multipart-post (2.4.1)
-    multiwoven-integrations (0.11.6)
+    multiwoven-integrations (0.15.0)
+      MailchimpMarketing
       activesupport
       async-websocket
       aws-sdk-athena
@@ -1962,10 +1967,11 @@ GEM
       sequel
       slack-ruby-client
       stripe
+      tiny_tds
       zendesk_api
     mutex_m (0.2.0)
     mysql2 (0.5.6)
-    net-http (0.4.1)
+    net-http (0.3.2)
       uri
     net-imap (0.4.9.1)
       date
@@ -2008,18 +2014,18 @@ GEM
       actionmailer (>= 3)
       net-smtp
       premailer (~> 1.7, >= 1.7.9)
-    process_executer (1.1.2)
+    process_executer (1.2.0)
     protocol-hpack (1.5.1)
-    protocol-http (0.37.0)
-    protocol-http1 (0.27.0)
+    protocol-http (0.44.0)
+    protocol-http1 (0.28.1)
       protocol-http (~> 0.22)
-    protocol-http2 (0.19.3)
+    protocol-http2 (0.20.0)
       protocol-hpack (~> 1.4)
       protocol-http (~> 0.18)
-    protocol-rack (0.10.0)
-      protocol-http (~> 0.37)
+    protocol-rack (0.11.0)
+      protocol-http (~> 0.43)
       rack (>= 1.0)
-    protocol-websocket (0.19.0)
+    protocol-websocket (0.20.1)
       protocol-http (~> 0.2)
     psych (5.1.2)
       stringio
@@ -2137,7 +2143,7 @@ GEM
     ruby-progressbar (1.13.0)
     ruby2_keywords (0.0.5)
     rubyzip (2.3.2)
-    sequel (5.85.0)
+    sequel (5.86.0)
       bigdecimal
     shoulda-matchers (5.3.0)
       activesupport (>= 5.2.0)
@@ -2160,15 +2166,16 @@ GEM
       faraday-multipart
       gli
       hashie
-    sorbet-runtime (0.5.11592)
+    sorbet-runtime (0.5.11647)
     stringio (3.1.0)
-    stripe (13.0.0)
+    stripe (13.1.1)
     strong_migrations (1.8.0)
       activerecord (>= 5.2)
     thor (1.3.0)
     timecop (0.9.8)
     timeout (0.4.1)
-    traces (0.13.1)
+    tiny_tds (2.1.7)
+    traces (0.14.1)
     trailblazer-option (0.1.2)
     typhoeus (1.4.1)
       ethon (>= 0.9.0)
@@ -2180,7 +2187,7 @@ GEM
     unf_ext (0.0.9.1)
     unicode-display_width (2.5.0)
     uniform_notifier (1.16.0)
-    uri (0.13.1)
+    uri (0.12.0)
     uri_template (0.7.0)
     warden (1.2.9)
       rack (>= 2.0.9)
@@ -2234,7 +2241,7 @@ DEPENDENCIES
   jwt
   kaminari
   liquid
-  multiwoven-integrations (~> 0.11.6)
+  multiwoven-integrations (~> 0.15.0)
   mysql2
   newrelic_rpm
   parallel

From f7dbfda67a4eae3ae7b86c913a0b015b2dd215a4 Mon Sep 17 00:00:00 2001
From: afthab vp <afthab@multiwoven.com>
Date: Thu, 14 Nov 2024 18:19:05 +0530
Subject: [PATCH 35/48] fix(CE): Dockerfile update free tds (#470)

---
 server/Dockerfile | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/server/Dockerfile b/server/Dockerfile
index 1615d389..4b7acd17 100644
--- a/server/Dockerfile
+++ b/server/Dockerfile
@@ -67,6 +67,15 @@ RUN apt-get update -qq && \
     apt-get install -y unixodbc && \
     rm -rf /var/lib/apt/lists/*
 
+# Install free tds for tiny tds gem dependency
+RUN apt-get install libc6-dev && \
+    wget http://www.freetds.org/files/stable/freetds-1.4.10.tar.gz && \
+    tar -xzf freetds-1.4.10.tar.gz && \
+    cd freetds-1.4.10 && \
+    ./configure --prefix=/usr/local --with-tdsver=7.4 && \
+    make && \
+    make install
+
 RUN apt-get update -qq && \
     apt-get install -y --no-install-recommends wget && \
     rm -rf /var/lib/apt/lists/*

From 5ebfcc8493effa033b2e3dc61db6f0e2e86bab52 Mon Sep 17 00:00:00 2001
From: Basil V Bose <bvb007@gmail.com>
Date: Wed, 13 Nov 2024 19:22:01 +0530
Subject: [PATCH 36/48] feat(CE): added dynamic_sql type to model (#576)

---
 server/app/contracts/model_contracts.rb       |  16 ++-
 server/app/models/model.rb                    |  25 ++++-
 .../models/configuration_aiml.json            |   9 ++
 .../models/configuration_dynamic_sql.json     |   9 ++
 server/spec/contracts/model_contracts_spec.rb |  88 ++++++++++++++-
 server/spec/models/model_spec.rb              | 103 ++++++++++++++++--
 .../requests/api/v1/models_controller_spec.rb |  70 +++++++++++-
 7 files changed, 304 insertions(+), 16 deletions(-)
 create mode 100644 server/app/models/schema_validations/models/configuration_aiml.json
 create mode 100644 server/app/models/schema_validations/models/configuration_dynamic_sql.json

diff --git a/server/app/contracts/model_contracts.rb b/server/app/contracts/model_contracts.rb
index a9d88e77..00c032cb 100644
--- a/server/app/contracts/model_contracts.rb
+++ b/server/app/contracts/model_contracts.rb
@@ -42,8 +42,20 @@ class Create < Dry::Validation::Contract
     end
 
     rule(model: :configuration) do
-      if values[:model][:configuration].blank? && (values[:model][:query_type] == "ai_ml")
-        key.failure("Configuration is required for this query type")
+      query_type = values[:model][:query_type]
+
+      if %w[dynamic_sql ai_ml].include? query_type
+        if value.blank?
+          key.failure("Configuration is required for this query type")
+        elsif query_type == "ai_ml"
+          key.failure("Config must contain harvester details") unless value.key?("harvesters")
+        else
+          key.failure("Config must contain harvester & json_schema") unless %w[harvesters
+                                                                               json_schema].any? do |k|
+                                                                              value.key?(k)
+                                                                            end
+
+        end
       end
     end
   end
diff --git a/server/app/models/model.rb b/server/app/models/model.rb
index 9a4f991c..38f6750f 100644
--- a/server/app/models/model.rb
+++ b/server/app/models/model.rb
@@ -15,15 +15,23 @@
 #  updated_at   :datetime         not null
 #
 class Model < ApplicationRecord
+  AI_ML_CONFIG_JSON_SCHEMA = Rails.root.join("app/models/schema_validations/models/configuration_aiml.json")
+  DYNAMIC_SQL_CONFIG_JSON_SCHEMA = Rails.root.join(
+    "app/models/schema_validations/models/configuration_dynamic_sql.json"
+  )
+
   validates :workspace_id, presence: true
   validates :connector_id, presence: true
   validates :name, presence: true
 
-  enum :query_type, %i[raw_sql dbt soql table_selector ai_ml]
+  enum :query_type, %i[raw_sql dbt soql table_selector ai_ml dynamic_sql]
 
   validates :query, presence: true, if: :requires_query?
   # Havesting configuration
   validates :configuration, presence: true, if: :requires_configuration?
+  validates :configuration, presence: true, json: { schema: lambda {
+                                                              configuration_schema_validation
+                                                            } }, if: :requires_configuration?
 
   belongs_to :workspace
   belongs_to :connector
@@ -31,10 +39,11 @@ class Model < ApplicationRecord
   has_many :syncs, dependent: :destroy
   has_many :visual_components, dependent: :destroy
 
-  scope :data, -> { where(query_type: %i[raw_sql dbt soql table_selector]) }
+  scope :data, -> { where(query_type: %i[raw_sql dbt soql table_selector dynamic_sql]) }
   scope :ai_ml, -> { where(query_type: :ai_ml) }
 
   default_scope { order(updated_at: :desc) }
+
   def to_protocol
     Multiwoven::Integrations::Protocol::Model.new(
       name:,
@@ -49,6 +58,16 @@ def requires_query?
   end
 
   def requires_configuration?
-    %w[ai_ml].include?(query_type)
+    %w[ai_ml dynamic_sql].include?(query_type)
+  end
+
+  private
+
+  def configuration_schema_validation
+    if ai_ml?
+      AI_ML_CONFIG_JSON_SCHEMA
+    elsif dynamic_sql?
+      DYNAMIC_SQL_CONFIG_JSON_SCHEMA
+    end
   end
 end
diff --git a/server/app/models/schema_validations/models/configuration_aiml.json b/server/app/models/schema_validations/models/configuration_aiml.json
new file mode 100644
index 00000000..8be3550f
--- /dev/null
+++ b/server/app/models/schema_validations/models/configuration_aiml.json
@@ -0,0 +1,9 @@
+{
+  "type": "object",
+  "$schema": "http://json-schema.org/draft-04/schema#",
+  "properties": {
+    "harvesters": { "type": "array" },
+    "json_schema": { "type": "array" }
+  },
+  "required": ["harvesters"]
+}
\ No newline at end of file
diff --git a/server/app/models/schema_validations/models/configuration_dynamic_sql.json b/server/app/models/schema_validations/models/configuration_dynamic_sql.json
new file mode 100644
index 00000000..f0b75d76
--- /dev/null
+++ b/server/app/models/schema_validations/models/configuration_dynamic_sql.json
@@ -0,0 +1,9 @@
+{
+  "type": "object",
+  "$schema": "http://json-schema.org/draft-04/schema#",
+  "properties": {
+    "harvesters": { "type": "array" },
+    "json_schema": { "type": "array" }
+  },
+  "required": ["harvesters", "json_schema"]
+}
\ No newline at end of file
diff --git a/server/spec/contracts/model_contracts_spec.rb b/server/spec/contracts/model_contracts_spec.rb
index e92c6489..dc77de5b 100644
--- a/server/spec/contracts/model_contracts_spec.rb
+++ b/server/spec/contracts/model_contracts_spec.rb
@@ -56,9 +56,43 @@
         }
       end
 
+      let(:valid_input_dynamic_sql) do
+        {
+          model: {
+            connector_id: 1,
+            name: "Model Name",
+            query: "SELECT * FROM table;",
+            query_type: "dynamic_sql",
+            primary_key: "id",
+            configuration: { "json_schema" => [], "harvesters" => [] }
+          }
+        }
+      end
+
+      let(:valid_input_ai_ml) do
+        {
+          model: {
+            connector_id: 1,
+            name: "Model Name",
+            query: "SELECT * FROM table;",
+            query_type: "dynamic_sql",
+            primary_key: "id",
+            configuration: { "harvesters" => [] }
+          }
+        }
+      end
+
       it "passes validation" do
         expect(contract.call(valid_inputs)).to be_success
       end
+
+      it "passes validation for dynamic_sql query_type" do
+        expect(contract.call(valid_input_dynamic_sql)).to be_success
+      end
+
+      it "passes validation for ai_ml query_type" do
+        expect(contract.call(valid_input_ai_ml)).to be_success
+      end
     end
 
     context "with missing query for query_type requiring it" do
@@ -118,7 +152,7 @@
       end
     end
 
-    context "with missing configuration for ai_ml query_type" do
+    context "with missing or invalid configuration for ai_ml query_type" do
       let(:invalid_inputs) do
         {
           model: {
@@ -130,10 +164,62 @@
         }
       end
 
+      let(:invalid_configuration) do
+        {
+          model: {
+            connector_id: 1,
+            name: "Model Name",
+            query_type: "ai_ml",
+            primary_key: "id",
+            configuration: { "test" => "new" }
+          }
+        }
+      end
+
       it "fails validation due to missing configuration" do
         result = contract.call(invalid_inputs)
         expect(result.errors[:model][:configuration]).to include("Configuration is required for this query type")
       end
+
+      it "fails validation due to invalid configuration" do
+        result = contract.call(invalid_configuration)
+        expect(result.errors[:model][:configuration]).to include("Config must contain harvester details")
+      end
+    end
+
+    context "with missing or invalid configuration for dynamic_sql query_type" do
+      let(:invalid_inputs) do
+        {
+          model: {
+            connector_id: 1,
+            name: "Model Name",
+            query_type: "dynamic_sql",
+            primary_key: "id"
+          }
+        }
+      end
+
+      let(:invalid_configuration) do
+        {
+          model: {
+            connector_id: 1,
+            name: "Model Name",
+            query_type: "dynamic_sql",
+            primary_key: "id",
+            configuration: { "test" => "new" }
+          }
+        }
+      end
+
+      it "fails validation due to missing configuration" do
+        result = contract.call(invalid_inputs)
+        expect(result.errors[:model][:configuration]).to include("Configuration is required for this query type")
+      end
+
+      it "fails validation due to invalid configuration" do
+        result = contract.call(invalid_configuration)
+        expect(result.errors[:model][:configuration]).to include("Config must contain harvester & json_schema")
+      end
     end
   end
 
diff --git a/server/spec/models/model_spec.rb b/server/spec/models/model_spec.rb
index b28d7a76..41ae8c90 100644
--- a/server/spec/models/model_spec.rb
+++ b/server/spec/models/model_spec.rb
@@ -53,7 +53,8 @@
           name: "test_model",
           query_type: :ai_ml, connector_id: source.id,
           workspace_id: source.workspace_id,
-          configuration: { "field1" => "value1" }
+          configuration: { harvesters:
+                     [{ "value" => "dynamic test", "method" => "dom", "selector" => "dom_id", "preprocess" => "" }] }
         )
         model.query = nil
         expect(model).to be_valid
@@ -62,13 +63,64 @@
 
     context "when query_type requires configuration" do
       it "validates presence of configuration" do
-        model = Model.new(
+        ai_ml_model = Model.new(
           name: "test_model",
           query_type: :ai_ml, connector_id: source.id,
           workspace_id: source.workspace_id
         )
-        model.configuration = nil
-        expect(model).not_to be_valid
+        dynamic_sql_model = Model.new(
+          name: "test_model",
+          query_type: :dynamic_sql, connector_id: source.id,
+          workspace_id: source.workspace_id
+        )
+        ai_ml_model.configuration = nil
+        dynamic_sql_model.configuration = nil
+        expect(ai_ml_model).not_to be_valid
+        expect(dynamic_sql_model).not_to be_valid
+      end
+
+      context "validates json schema of configuration" do
+        context "validates json schema of ai_ml models" do
+          it "is does returns invalid for ai ml models without valid configuration" do
+            ai_ml_model = Model.new(
+              name: "test_model",
+              query_type: :ai_ml, connector_id: source.id,
+              workspace_id: source.workspace_id,
+              configuration: { "harvesters": { "wrong": "format" } }
+            )
+            expect(ai_ml_model).not_to be_valid
+          end
+          it "is does returns valid for ai ml models with valid configuration" do
+            ai_ml_model = Model.new(
+              name: "test_model",
+              query_type: :ai_ml, connector_id: source.id,
+              workspace_id: source.workspace_id,
+              configuration: { "harvesters": [] }
+            )
+            expect(ai_ml_model).to be_valid
+          end
+        end
+
+        context "validates json schema of dynamic_sql models" do
+          it "is does returns invalid for dyanmic models without valid configuration" do
+            dynamic_sql_model = Model.new(
+              name: "test_model",
+              query_type: :dynamic_sql, connector_id: source.id,
+              workspace_id: source.workspace_id,
+              configuration: { "harvesters": { "wrong": "format" } }
+            )
+            expect(dynamic_sql_model).not_to be_valid
+          end
+          it "is does returns valid for ai ml models with valid configuration" do
+            dynamic_sql_model = Model.new(
+              name: "test_model",
+              query_type: :dynamic_sql, connector_id: source.id,
+              workspace_id: source.workspace_id,
+              configuration: { "harvesters": [], "json_schema": [] }
+            )
+            expect(dynamic_sql_model).to be_valid
+          end
+        end
       end
     end
 
@@ -128,7 +180,8 @@
 
   describe "query_type" do
     it "defines query_type enum with specified values" do
-      expect(Model.query_types).to eq({ "raw_sql" => 0, "dbt" => 1, "soql" => 2, "table_selector" => 3, "ai_ml" => 4 })
+      expect(Model.query_types).to eq({ "raw_sql" => 0, "dbt" => 1, "soql" => 2, "table_selector" => 3, "ai_ml" => 4,
+                                        "dynamic_sql" => 5 })
     end
   end
 
@@ -138,12 +191,48 @@
     let!(:dbt_model) { create(:model, query_type: :dbt, connector: source) }
     let!(:soql_model) { create(:model, query_type: :soql, connector: source) }
     let!(:table_selector_model) { create(:model, query_type: :table_selector, connector: source) }
-    let!(:ai_ml_model) { create(:model, query_type: :ai_ml, connector: source, configuration: { test: "value" }) }
+    let!(:ai_ml_model) do
+      create(:model, query_type: :ai_ml, connector: source,
+                     configuration: { harvesters:
+                     [{ "value" => "dynamic test", "method" => "dom", "selector" => "dom_id", "preprocess" => "" }] })
+    end
+    let!(:dynamic_sql_model) do
+      create(:model, query_type: :dynamic_sql, connector: source,
+                     configuration:
+                      {
+                        "harvesters": [
+                          {
+                            "value": "dynamic test",
+                            "method": "dom",
+                            "selector": "dom_id",
+                            "preprocess": ""
+                          }
+                        ],
+                        "json_schema": [
+                          {
+                            "input": [
+                              {
+                                "name": "risk_level",
+                                "type": "string",
+                                "value": "",
+                                "value_type": "dynamic"
+                              }
+                            ],
+                            "output": [
+                              {
+                                "name": "data.col0.calculated_risk",
+                                "type": "string"
+                              }
+                            ]
+                          }
+                        ]
+                      })
+    end
 
     describe ".data" do
       it "returns models with query_type in [raw_sql, dbt, soql, table_selector]" do
         data_models = Model.data
-        expect(data_models).to include(raw_sql_model, dbt_model, soql_model, table_selector_model)
+        expect(data_models).to include(raw_sql_model, dbt_model, soql_model, table_selector_model, dynamic_sql_model)
         expect(data_models).not_to include(ai_ml_model)
       end
     end
diff --git a/server/spec/requests/api/v1/models_controller_spec.rb b/server/spec/requests/api/v1/models_controller_spec.rb
index 7cb73a4f..0ab2da9f 100644
--- a/server/spec/requests/api/v1/models_controller_spec.rb
+++ b/server/spec/requests/api/v1/models_controller_spec.rb
@@ -24,7 +24,7 @@
   let!(:dbt_model) { create(:model, query_type: :dbt, connector:, workspace:) }
   let!(:soql_model) { create(:model, query_type: :soql, connector:, workspace:) }
   let!(:ai_ml_model) do
-    create(:model, query_type: :ai_ml, connector:, configuration: { key: "value" }, workspace:)
+    create(:model, query_type: :ai_ml, connector:, configuration: { harvesters: [] }, workspace:)
   end
   let(:viewer_role) { create(:role, :viewer) }
   let(:member_role) { create(:role, :member) }
@@ -351,7 +351,7 @@
               name: "AI/ML Model",
               query_type: "ai_ml",
               primary_key: "id",
-              configuration: { "test" => "value" }
+              configuration: { "harvesters" => [] }
             }
           }
         end
@@ -379,6 +379,70 @@
         end
       end
 
+      context "when creating a model with query_type dynamic_sql with configuration present" do
+        let(:request_body) do
+          {
+            model: {
+              connector_id: models.first.connector_id,
+              name: "Dynamic SQL Model",
+              query_type: "dynamic_sql",
+              primary_key: "id",
+              configuration:
+              {
+                "harvesters" => [
+                  {
+                    "value" => "dynamic test",
+                    "method" => "dom",
+                    "selector" => "dom_id",
+                    "preprocess" => ""
+                  }
+                ],
+                "json_schema" => [
+                  {
+                    "input" => [
+                      {
+                        "name" => "risk_level",
+                        "type" => "string",
+                        "value" => "",
+                        "value_type" => "dynamic"
+                      }
+                    ],
+                    "output" => [
+                      {
+                        "name" => "data.col0.calculated_risk",
+                        "type" => "string"
+                      }
+                    ]
+                  }
+                ]
+              }
+            }
+          }
+        end
+
+        it "creates the model and returns success" do
+          post "/api/v1/models", params: request_body.to_json, headers: { "Content-Type": "application/json" }
+            .merge(auth_headers(user, workspace_id))
+          expect(response).to have_http_status(:created)
+          response_hash = JSON.parse(response.body).with_indifferent_access
+          expect(response_hash.dig(:data, :id)).to be_present
+          expect(response_hash.dig(:data, :attributes, :name)).to eq(request_body.dig(:model, :name))
+          expect(response_hash.dig(:data, :attributes, :query_type)).to eq("dynamic_sql")
+          expect(response_hash.dig(:data, :attributes, :configuration)).to eq(request_body.dig(:model, :configuration))
+
+          audit_log = AuditLog.last
+          expect(audit_log).not_to be_nil
+          expect(audit_log.user_id).to eq(user.id)
+          expect(audit_log.action).to eq("create")
+          expect(audit_log.resource_type).to eq("Model")
+          expect(audit_log.resource_id).to eq(nil)
+          expect(audit_log.resource).to eq(request_body.dig(:model, :name))
+          expect(audit_log.workspace_id).to eq(workspace.id)
+          expect(audit_log.created_at).not_to be_nil
+          expect(audit_log.updated_at).not_to be_nil
+        end
+      end
+
       it "returns fail viwer role" do
         workspace.workspace_users.first.update(role: viewer_role)
         post "/api/v1/models", params: request_body.to_json, headers: { "Content-Type": "application/json" }
@@ -499,7 +563,7 @@
               name: "Updated AI/ML Model",
               query_type: "ai_ml",
               primary_key: "updated_id",
-              configuration: { "updated_test" => "value" }
+              configuration: { "harvesters" => [] }
             }
           }
         end

From 26eef75fbd0c2f1c6091f3d9bca000d89b19b225 Mon Sep 17 00:00:00 2001
From: "github-actions[bot]"
 <41898282+github-actions[bot]@users.noreply.github.com>
Date: Thu, 14 Nov 2024 08:07:05 -0500
Subject: [PATCH 37/48] chore(CE): Change http model name (#469)

Co-authored-by: TivonB-AI2
---
 integrations/lib/multiwoven/integrations/rollout.rb           | 2 +-
 .../integrations/source/http_model/config/meta.json           | 2 +-
 .../integrations/source/http_model/config/spec.json           | 4 ++--
 3 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/integrations/lib/multiwoven/integrations/rollout.rb b/integrations/lib/multiwoven/integrations/rollout.rb
index 6d9488b8..44c12bb9 100644
--- a/integrations/lib/multiwoven/integrations/rollout.rb
+++ b/integrations/lib/multiwoven/integrations/rollout.rb
@@ -2,7 +2,7 @@
 
 module Multiwoven
   module Integrations
-    VERSION = "0.15.0"
+    VERSION = "0.15.1"
 
     ENABLED_SOURCES = %w[
       Snowflake
diff --git a/integrations/lib/multiwoven/integrations/source/http_model/config/meta.json b/integrations/lib/multiwoven/integrations/source/http_model/config/meta.json
index dca2f332..f1e85784 100644
--- a/integrations/lib/multiwoven/integrations/source/http_model/config/meta.json
+++ b/integrations/lib/multiwoven/integrations/source/http_model/config/meta.json
@@ -1,7 +1,7 @@
 {
   "data": {
     "name": "HttpModel",
-    "title": "HTTP Model",
+    "title": "HTTP Model Endpoint",
     "connector_type": "source",
     "category": "AI Model",
     "documentation_url": "https://docs.mutliwoven.com",
diff --git a/integrations/lib/multiwoven/integrations/source/http_model/config/spec.json b/integrations/lib/multiwoven/integrations/source/http_model/config/spec.json
index 4d393ec9..e504850f 100644
--- a/integrations/lib/multiwoven/integrations/source/http_model/config/spec.json
+++ b/integrations/lib/multiwoven/integrations/source/http_model/config/spec.json
@@ -1,10 +1,10 @@
 {
-  "documentation_url": "https://docs.multiwoven.com/integrations/source/http-model",
+  "documentation_url": "https://docs.multiwoven.com/integrations/source/http-model-endpoint",
   "stream_type": "user_defined",
   "connector_query_type": "ai_ml",
   "connection_specification": {
     "$schema": "http://json-schema.org/draft-07/schema#",
-    "title": "Http Model",
+    "title": "Http Model Endpoint",
     "type": "object",
     "required": ["url_host"],
     "properties": {

From 435ff5067773b26851831db946c2b0ba1ba41ed4 Mon Sep 17 00:00:00 2001
From: Basil V Bose <bvb007@gmail.com>
Date: Thu, 14 Nov 2024 11:54:00 +0530
Subject: [PATCH 38/48] feat(CE): Fetch json_schema from model (#581)

---
 server/app/models/catalog.rb                  |  5 ++
 .../models/configuration_aiml.json            |  2 +-
 .../models/configuration_dynamic_sql.json     |  2 +-
 server/app/serializers/model_serializer.rb    | 12 +++
 server/spec/contracts/model_contracts_spec.rb |  2 +-
 server/spec/factories/catalogs.rb             |  9 +-
 server/spec/models/catalog_spec.rb            | 25 ++++++
 server/spec/models/model_spec.rb              |  5 +-
 .../requests/api/v1/models_controller_spec.rb | 82 +++++++++++++++++--
 9 files changed, 128 insertions(+), 16 deletions(-)

diff --git a/server/app/models/catalog.rb b/server/app/models/catalog.rb
index 322c7ea9..095b2ddf 100644
--- a/server/app/models/catalog.rb
+++ b/server/app/models/catalog.rb
@@ -50,4 +50,9 @@ def stream_to_protocol(stream)
       request_rate_concurrency:
     )
   end
+
+  def json_schema(stream_name)
+    stream = find_stream_by_name(stream_name)
+    stream&.[]("json_schema") || {}
+  end
 end
diff --git a/server/app/models/schema_validations/models/configuration_aiml.json b/server/app/models/schema_validations/models/configuration_aiml.json
index 8be3550f..764e360d 100644
--- a/server/app/models/schema_validations/models/configuration_aiml.json
+++ b/server/app/models/schema_validations/models/configuration_aiml.json
@@ -3,7 +3,7 @@
   "$schema": "http://json-schema.org/draft-04/schema#",
   "properties": {
     "harvesters": { "type": "array" },
-    "json_schema": { "type": "array" }
+    "json_schema": { "type": "object" }
   },
   "required": ["harvesters"]
 }
\ No newline at end of file
diff --git a/server/app/models/schema_validations/models/configuration_dynamic_sql.json b/server/app/models/schema_validations/models/configuration_dynamic_sql.json
index f0b75d76..36a53157 100644
--- a/server/app/models/schema_validations/models/configuration_dynamic_sql.json
+++ b/server/app/models/schema_validations/models/configuration_dynamic_sql.json
@@ -3,7 +3,7 @@
   "$schema": "http://json-schema.org/draft-04/schema#",
   "properties": {
     "harvesters": { "type": "array" },
-    "json_schema": { "type": "array" }
+    "json_schema": { "type": "object" }
   },
   "required": ["harvesters", "json_schema"]
 }
\ No newline at end of file
diff --git a/server/app/serializers/model_serializer.rb b/server/app/serializers/model_serializer.rb
index eb11b504..071e85f1 100644
--- a/server/app/serializers/model_serializer.rb
+++ b/server/app/serializers/model_serializer.rb
@@ -6,4 +6,16 @@ class ModelSerializer < ActiveModel::Serializer
   attribute :connector do
     ConnectorSerializer.new(object.connector).attributes
   end
+
+  def configuration
+    if object.ai_ml?
+      connector = object.connector
+      json_schema = connector.catalog.json_schema(connector.connector_name)
+      existing_config = object.configuration
+
+      existing_config.merge({ json_schema: })
+    else
+      object.configuration
+    end
+  end
 end
diff --git a/server/spec/contracts/model_contracts_spec.rb b/server/spec/contracts/model_contracts_spec.rb
index dc77de5b..9cbbe464 100644
--- a/server/spec/contracts/model_contracts_spec.rb
+++ b/server/spec/contracts/model_contracts_spec.rb
@@ -64,7 +64,7 @@
             query: "SELECT * FROM table;",
             query_type: "dynamic_sql",
             primary_key: "id",
-            configuration: { "json_schema" => [], "harvesters" => [] }
+            configuration: { "json_schema" => {}, "harvesters" => [] }
           }
         }
       end
diff --git a/server/spec/factories/catalogs.rb b/server/spec/factories/catalogs.rb
index 29d38326..0569484f 100644
--- a/server/spec/factories/catalogs.rb
+++ b/server/spec/factories/catalogs.rb
@@ -20,7 +20,14 @@
       {
         "streams" => [
           { "name" => "profile", "batch_support" => false, "batch_size" => 1, "json_schema" => {} },
-          { "name" => "customer", "batch_support" => false, "batch_size" => 1, "json_schema" => {} }
+          { "name" => "customer", "batch_support" => false, "batch_size" => 1, "json_schema" => {} },
+          { "name" => "DatabricksModel", "batch_support" => false, "batch_size" => 1, "json_schema" =>
+            {
+              "input" => [{ "name" => "inputs.0", "type" => "string", "value" => "dynamic", "value_type" => "dynamic" },
+                          { "name" => "inputs.0", "type" => "number", "value" => "9522", "value_type" => "static" }],
+              "output" => [{ "name" => "predictions.col1.0", "type" => "string" },
+                           { "name" => "predictions.col1.1", "type" => "number" }]
+            } }
         ],
         "request_rate_limit" => 60,
         "request_rate_limit_unit" => "minute",
diff --git a/server/spec/models/catalog_spec.rb b/server/spec/models/catalog_spec.rb
index 5ca6e36d..6fda866d 100644
--- a/server/spec/models/catalog_spec.rb
+++ b/server/spec/models/catalog_spec.rb
@@ -101,4 +101,29 @@
       end
     end
   end
+
+  describe "#json_schema" do
+    let(:json_schema) do
+      { "input" => [{ "name" => "inputs.0", "type" => "string", "value" => "dynamic", "value_type" => "dynamic" },
+                    { "name" => "inputs.0", "type" => "number", "value" => "9522", "value_type" => "static" }],
+        "output" => [{ "name" => "predictions.col1.0", "type" => "string" },
+                     { "name" => "predictions.col1.1", "type" => "number" }] }
+    end
+    let(:catalog) do
+      create(:catalog, catalog: { "streams" => [{
+               "name" => "DatabricksModel",
+               "json_schema" => json_schema
+             }] })
+    end
+
+    it "returns the correct json_schema when it exists" do
+      json_schema_from_catalog = catalog.json_schema("DatabricksModel")
+      expect(json_schema_from_catalog).to eq(json_schema)
+    end
+
+    it "returns an empty json when no valid schema available" do
+      json_schema_from_catalog = catalog.json_schema("NotExist")
+      expect(json_schema_from_catalog).to eq({})
+    end
+  end
 end
diff --git a/server/spec/models/model_spec.rb b/server/spec/models/model_spec.rb
index 41ae8c90..ab801413 100644
--- a/server/spec/models/model_spec.rb
+++ b/server/spec/models/model_spec.rb
@@ -116,7 +116,7 @@
               name: "test_model",
               query_type: :dynamic_sql, connector_id: source.id,
               workspace_id: source.workspace_id,
-              configuration: { "harvesters": [], "json_schema": [] }
+              configuration: { "harvesters": [], "json_schema": {} }
             )
             expect(dynamic_sql_model).to be_valid
           end
@@ -208,7 +208,7 @@
                             "preprocess": ""
                           }
                         ],
-                        "json_schema": [
+                        "json_schema":
                           {
                             "input": [
                               {
@@ -225,7 +225,6 @@
                               }
                             ]
                           }
-                        ]
                       })
     end
 
diff --git a/server/spec/requests/api/v1/models_controller_spec.rb b/server/spec/requests/api/v1/models_controller_spec.rb
index 0ab2da9f..d9fa828f 100644
--- a/server/spec/requests/api/v1/models_controller_spec.rb
+++ b/server/spec/requests/api/v1/models_controller_spec.rb
@@ -10,6 +10,10 @@
     create(:connector, workspace:, connector_type: "destination", name: "klavio1", connector_name: "Klaviyo")
   end
 
+  let(:source_connector) do
+    create(:connector, workspace:, connector_type: "source", name: "DatabricksModel", connector_name: "DatabricksModel")
+  end
+
   let(:connector_without_catalog) do
     create(:connector, workspace:, connector_type: "destination", name: "klavio1", connector_name: "Klaviyo")
   end
@@ -26,11 +30,15 @@
   let!(:ai_ml_model) do
     create(:model, query_type: :ai_ml, connector:, configuration: { harvesters: [] }, workspace:)
   end
+  let!(:ai_ml_source_model) do
+    create(:model, query_type: :ai_ml, connector: source_connector, configuration: { harvesters: [] }, workspace:)
+  end
   let(:viewer_role) { create(:role, :viewer) }
   let(:member_role) { create(:role, :member) }
 
   before do
     create(:catalog, connector:)
+    create(:catalog, connector: source_connector)
     user.confirm
   end
 
@@ -47,7 +55,7 @@
         get "/api/v1/models", headers: auth_headers(user, workspace_id)
         expect(response).to have_http_status(:ok)
         response_hash = JSON.parse(response.body).with_indifferent_access
-        expect(response_hash[:data].count).to eql(6)
+        expect(response_hash[:data].count).to eql(7)
         expect(response_hash.dig(:data, 0, :type)).to eq("models")
         expect(response_hash.dig(:links, :first)).to include("http://www.example.com/api/v1/models?page=1")
 
@@ -68,7 +76,7 @@
         get "/api/v1/models", headers: auth_headers(user, workspace_id)
         expect(response).to have_http_status(:ok)
         response_hash = JSON.parse(response.body).with_indifferent_access
-        expect(response_hash[:data].count).to eql(6)
+        expect(response_hash[:data].count).to eql(7)
         expect(response_hash.dig(:data, 0, :type)).to eq("models")
         expect(response_hash.dig(:links, :first)).to include("http://www.example.com/api/v1/models?page=1")
       end
@@ -78,7 +86,7 @@
         get "/api/v1/models", headers: auth_headers(user, workspace_id)
         expect(response).to have_http_status(:ok)
         response_hash = JSON.parse(response.body).with_indifferent_access
-        expect(response_hash[:data].count).to eql(6)
+        expect(response_hash[:data].count).to eql(7)
         expect(response_hash.dig(:data, 0, :type)).to eq("models")
         expect(response_hash.dig(:links, :first)).to include("http://www.example.com/api/v1/models?page=1")
 
@@ -134,7 +142,7 @@
         get "/api/v1/models", headers: auth_headers(user, workspace_id)
         expect(response).to have_http_status(:ok)
         model_ids = JSON.parse(response.body)["data"].map { |m| m["id"] }
-        expect(model_ids.count).to eql(6)
+        expect(model_ids.count).to eql(7)
 
         audit_log = AuditLog.last
         expect(audit_log).not_to be_nil
@@ -183,6 +191,34 @@
         expect(audit_log.updated_at).not_to be_nil
       end
 
+      it "returns success and fetch ai_ml model with configuration from catalog" do
+        get "/api/v1/models/#{ai_ml_source_model.id}", headers: auth_headers(user, workspace_id)
+        expect(response).to have_http_status(:ok)
+        response_hash = JSON.parse(response.body).with_indifferent_access
+        expect(response_hash.dig(:data, :id)).to be_present
+        expect(response_hash.dig(:data, :id)).to eq(ai_ml_source_model.id.to_s)
+        expect(response_hash.dig(:data, :type)).to eq("models")
+        expect(response_hash.dig(:data, :attributes, :name)).to eq(ai_ml_source_model.name)
+        expect(response_hash.dig(:data, :attributes, :query)).to eq(ai_ml_source_model.query)
+        expect(response_hash.dig(:data, :attributes, :query_type)).to eq(ai_ml_source_model.query_type)
+        expect(response_hash.dig(:data, :attributes, :primary_key)).to eq(ai_ml_source_model.primary_key)
+        expected_configuration = {
+          "harvesters" => [],
+          "json_schema" =>
+          {
+            "input" => [
+              { "name" => "inputs.0", "type" => "string", "value" => "dynamic", "value_type" => "dynamic" },
+              { "name" => "inputs.0", "type" => "number", "value" => "9522", "value_type" => "static" }
+            ],
+            "output" => [
+              { "name" => "predictions.col1.0", "type" => "string" },
+              { "name" => "predictions.col1.1", "type" => "number" }
+            ]
+          }
+        }
+        expect(response_hash.dig(:data, :attributes, :configuration)).to eq(expected_configuration)
+      end
+
       it "returns success and fetch model for viewer role" do
         workspace.workspace_users.first.update(role: viewer_role)
         get "/api/v1/models/#{models.first.id}", headers: auth_headers(user, workspace_id)
@@ -364,7 +400,8 @@
           expect(response_hash.dig(:data, :id)).to be_present
           expect(response_hash.dig(:data, :attributes, :name)).to eq(request_body.dig(:model, :name))
           expect(response_hash.dig(:data, :attributes, :query_type)).to eq("ai_ml")
-          expect(response_hash.dig(:data, :attributes, :configuration)).to eq(request_body.dig(:model, :configuration))
+          expected_configuration = { "harvesters" => [], "json_schema" => {} }
+          expect(response_hash.dig(:data, :attributes, :configuration)).to eq(expected_configuration)
 
           audit_log = AuditLog.last
           expect(audit_log).not_to be_nil
@@ -397,7 +434,7 @@
                     "preprocess" => ""
                   }
                 ],
-                "json_schema" => [
+                "json_schema" =>
                   {
                     "input" => [
                       {
@@ -414,7 +451,6 @@
                       }
                     ]
                   }
-                ]
               }
             }
           }
@@ -428,7 +464,34 @@
           expect(response_hash.dig(:data, :id)).to be_present
           expect(response_hash.dig(:data, :attributes, :name)).to eq(request_body.dig(:model, :name))
           expect(response_hash.dig(:data, :attributes, :query_type)).to eq("dynamic_sql")
-          expect(response_hash.dig(:data, :attributes, :configuration)).to eq(request_body.dig(:model, :configuration))
+          expected_configuration = {
+            "harvesters" => [
+              {
+                "value" => "dynamic test",
+                "method" => "dom",
+                "selector" => "dom_id",
+                "preprocess" => ""
+              }
+            ],
+            "json_schema" =>
+              {
+                "input" => [
+                  {
+                    "name" => "risk_level",
+                    "type" => "string",
+                    "value" => "",
+                    "value_type" => "dynamic"
+                  }
+                ],
+                "output" => [
+                  {
+                    "name" => "data.col0.calculated_risk",
+                    "type" => "string"
+                  }
+                ]
+              }
+          }
+          expect(response_hash.dig(:data, :attributes, :configuration)).to eq(expected_configuration)
 
           audit_log = AuditLog.last
           expect(audit_log).not_to be_nil
@@ -576,7 +639,8 @@
           expect(response_hash.dig(:data, :id)).to eq(models.second.id.to_s)
           expect(response_hash.dig(:data, :attributes, :name)).to eq(request_body.dig(:model, :name))
           expect(response_hash.dig(:data, :attributes, :query_type)).to eq("ai_ml")
-          expect(response_hash.dig(:data, :attributes, :configuration)).to eq(request_body.dig(:model, :configuration))
+          expected_configuration = { "harvesters" => [], "json_schema" => {} }
+          expect(response_hash.dig(:data, :attributes, :configuration)).to eq(expected_configuration)
 
           audit_log = AuditLog.last
           expect(audit_log).not_to be_nil

From 234c1d8aa8accb71187ee679c3faed47fa12f65b Mon Sep 17 00:00:00 2001
From: "github-actions[bot]"
 <41898282+github-actions[bot]@users.noreply.github.com>
Date: Fri, 15 Nov 2024 09:13:38 -0500
Subject: [PATCH 39/48] chore(CE): Update HTTP Model spec (#583) (#474)

Co-authored-by: TivonB-AI2
---
 integrations/Gemfile.lock                           |  2 +-
 integrations/lib/multiwoven/integrations/rollout.rb |  2 +-
 .../integrations/source/http_model/config/spec.json | 13 +++++++------
 3 files changed, 9 insertions(+), 8 deletions(-)

diff --git a/integrations/Gemfile.lock b/integrations/Gemfile.lock
index 80bacca5..5a5979a7 100644
--- a/integrations/Gemfile.lock
+++ b/integrations/Gemfile.lock
@@ -7,7 +7,7 @@ GIT
 PATH
   remote: .
   specs:
-    multiwoven-integrations (0.15.0)
+    multiwoven-integrations (0.15.2)
       MailchimpMarketing
       activesupport
       async-websocket
diff --git a/integrations/lib/multiwoven/integrations/rollout.rb b/integrations/lib/multiwoven/integrations/rollout.rb
index 44c12bb9..b694772b 100644
--- a/integrations/lib/multiwoven/integrations/rollout.rb
+++ b/integrations/lib/multiwoven/integrations/rollout.rb
@@ -2,7 +2,7 @@
 
 module Multiwoven
   module Integrations
-    VERSION = "0.15.1"
+    VERSION = "0.15.2"
 
     ENABLED_SOURCES = %w[
       Snowflake
diff --git a/integrations/lib/multiwoven/integrations/source/http_model/config/spec.json b/integrations/lib/multiwoven/integrations/source/http_model/config/spec.json
index e504850f..557bef93 100644
--- a/integrations/lib/multiwoven/integrations/source/http_model/config/spec.json
+++ b/integrations/lib/multiwoven/integrations/source/http_model/config/spec.json
@@ -4,7 +4,7 @@
   "connector_query_type": "ai_ml",
   "connection_specification": {
     "$schema": "http://json-schema.org/draft-07/schema#",
-    "title": "Http Model Endpoint",
+    "title": "HTTP Model Endpoint",
     "type": "object",
     "required": ["url_host"],
     "properties": {
@@ -14,7 +14,8 @@
         "order": 0
       },
       "headers": {
-        "title": "Http Headers",
+        "title": "HTTP Headers",
+        "description": "Custom headers to include in the HTTP request. Useful for authentication, content type specifications, and other request metadata.",
         "order": 1,
         "additionalProperties": {
           "type": "string"
@@ -25,10 +26,10 @@
         "type": "object",
         "properties": {
           "timeout": {
-            "type": "integer",
-            "minimum": 0,
-            "default": 30,
-            "title": "Http Timeout",
+            "type": "string",
+            "default": "30",
+            "title": "HTTP Timeout",
+            "description": "The maximum time, in seconds, to wait for a response from the server before the request is canceled.",
             "order": 0
           }
         },

From ebfdd352c896ad9c2e3d7b88a2f58970d74c6370 Mon Sep 17 00:00:00 2001
From: "github-actions[bot]"
 <41898282+github-actions[bot]@users.noreply.github.com>
Date: Fri, 15 Nov 2024 10:05:43 -0500
Subject: [PATCH 40/48] Resolve conflict in cherry-pick of
 d10fecacbae9908b9ef7e8d08bb1fd3bde1e3ef4 and change the commit message (#472)

Co-authored-by: TivonB-AI2
---
 .../lib/multiwoven/integrations/core/http_client.rb        | 7 +++++--
 integrations/lib/multiwoven/integrations/rollout.rb        | 2 +-
 2 files changed, 6 insertions(+), 3 deletions(-)

diff --git a/integrations/lib/multiwoven/integrations/core/http_client.rb b/integrations/lib/multiwoven/integrations/core/http_client.rb
index c1e0939b..0e36b49a 100644
--- a/integrations/lib/multiwoven/integrations/core/http_client.rb
+++ b/integrations/lib/multiwoven/integrations/core/http_client.rb
@@ -10,8 +10,11 @@ def request(url, method, payload: nil, headers: {}, config: {})
           http.use_ssl = (uri.scheme == "https")
 
           # Set timeout if provided
-          http.open_timeout = config[:timeout] if config[:timeout]
-          http.read_timeout = config[:timeout] if config[:timeout]
+          if config[:timeout]
+            timeout_value = config[:timeout].to_f
+            http.open_timeout = timeout_value
+            http.read_timeout = timeout_value
+          end
 
           request = build_request(method, uri, payload, headers)
           http.request(request)
diff --git a/integrations/lib/multiwoven/integrations/rollout.rb b/integrations/lib/multiwoven/integrations/rollout.rb
index b694772b..3c69f4f9 100644
--- a/integrations/lib/multiwoven/integrations/rollout.rb
+++ b/integrations/lib/multiwoven/integrations/rollout.rb
@@ -2,7 +2,7 @@
 
 module Multiwoven
   module Integrations
-    VERSION = "0.15.2"
+    VERSION = "0.15.3"
 
     ENABLED_SOURCES = %w[
       Snowflake

From 17b34d6ef9d76715d1dbe01bb467bb5c3792df5a Mon Sep 17 00:00:00 2001
From: "github-actions[bot]"
 <41898282+github-actions[bot]@users.noreply.github.com>
Date: Fri, 15 Nov 2024 10:06:20 -0500
Subject: [PATCH 41/48] chore(CE): Server Gem Update 0.15.3 (#589) (#471)

Co-authored-by: TivonB-AI2
---
 server/Gemfile      |  2 +-
 server/Gemfile.lock | 12 ++++++------
 2 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/server/Gemfile b/server/Gemfile
index 37a78cae..625d9e0e 100644
--- a/server/Gemfile
+++ b/server/Gemfile
@@ -13,7 +13,7 @@ gem "interactor", "~> 3.0"
 
 gem "ruby-odbc", git: "https://github.com/Multiwoven/ruby-odbc.git"
 
-gem "multiwoven-integrations", "~> 0.15.0"
+gem "multiwoven-integrations", "~> 0.15.3"
 
 gem "temporal-ruby", github: "coinbase/temporal-ruby"
 
diff --git a/server/Gemfile.lock b/server/Gemfile.lock
index f2866920..b0dacc4f 100644
--- a/server/Gemfile.lock
+++ b/server/Gemfile.lock
@@ -125,7 +125,7 @@ GEM
       protocol-http1 (>= 0.28.1)
       protocol-http2 (~> 0.19)
       traces (~> 0.10)
-    async-pool (0.10.1)
+    async-pool (0.10.2)
       async (>= 1.25)
       traces
     async-websocket (0.30.0)
@@ -1809,7 +1809,7 @@ GEM
       retriable (>= 2.0, < 4.a)
     google-apis-sheets_v4 (0.38.0)
       google-apis-core (>= 0.15.0, < 2.a)
-    google-cloud-ai_platform-v1 (0.55.0)
+    google-cloud-ai_platform-v1 (0.57.0)
       gapic-common (>= 0.21.1, < 2.a)
       google-cloud-errors (~> 1.0)
       google-cloud-location (>= 0.7, < 2.a)
@@ -1935,7 +1935,7 @@ GEM
     msgpack (1.7.2)
     multi_json (1.15.0)
     multipart-post (2.4.1)
-    multiwoven-integrations (0.15.0)
+    multiwoven-integrations (0.15.3)
       MailchimpMarketing
       activesupport
       async-websocket
@@ -1971,7 +1971,7 @@ GEM
       zendesk_api
     mutex_m (0.2.0)
     mysql2 (0.5.6)
-    net-http (0.3.2)
+    net-http (0.5.0)
       uri
     net-imap (0.4.9.1)
       date
@@ -2187,7 +2187,7 @@ GEM
     unf_ext (0.0.9.1)
     unicode-display_width (2.5.0)
     uniform_notifier (1.16.0)
-    uri (0.12.0)
+    uri (1.0.2)
     uri_template (0.7.0)
     warden (1.2.9)
       rack (>= 2.0.9)
@@ -2241,7 +2241,7 @@ DEPENDENCIES
   jwt
   kaminari
   liquid
-  multiwoven-integrations (~> 0.15.0)
+  multiwoven-integrations (~> 0.15.3)
   mysql2
   newrelic_rpm
   parallel

From a7cb638cb5a44a84d0b042ebc58daa5b9a828e47 Mon Sep 17 00:00:00 2001
From: "github-actions[bot]"
 <41898282+github-actions[bot]@users.noreply.github.com>
Date: Mon, 18 Nov 2024 20:15:25 +0530
Subject: [PATCH 42/48] feat(CE): fetch data for dsql models (#596) (#476)

Co-authored-by: Basil V Bose <bvb007@gmail.com>
---
 server/app/models/model.rb                    |   4 +
 server/lib/utils/payload_generator/ai_ml.rb   |  58 ++++++++
 .../utils/payload_generator/dynamic_sql.rb    |  21 +++
 .../lib/utils/payload_generator/ai_ml_spec.rb | 127 ++++++++++++++++++
 .../payload_generator/dynamic_sql_spec.rb     |  40 ++++++
 5 files changed, 250 insertions(+)
 create mode 100644 server/lib/utils/payload_generator/ai_ml.rb
 create mode 100644 server/lib/utils/payload_generator/dynamic_sql.rb
 create mode 100644 server/spec/lib/utils/payload_generator/ai_ml_spec.rb
 create mode 100644 server/spec/lib/utils/payload_generator/dynamic_sql_spec.rb

diff --git a/server/app/models/model.rb b/server/app/models/model.rb
index 38f6750f..95584b5a 100644
--- a/server/app/models/model.rb
+++ b/server/app/models/model.rb
@@ -61,6 +61,10 @@ def requires_configuration?
     %w[ai_ml dynamic_sql].include?(query_type)
   end
 
+  def json_schema
+    configuration["json_schema"]
+  end
+
   private
 
   def configuration_schema_validation
diff --git a/server/lib/utils/payload_generator/ai_ml.rb b/server/lib/utils/payload_generator/ai_ml.rb
new file mode 100644
index 00000000..b1615793
--- /dev/null
+++ b/server/lib/utils/payload_generator/ai_ml.rb
@@ -0,0 +1,58 @@
+# frozen_string_literal: true
+
+module Utils
+  module PayloadGenerator
+    module AiMl
+      def self.generate_payload(configuration, harvest_values)
+        result = {}
+
+        configuration.each do |config|
+          path = config["name"].split(".")
+          value = fetch_value(config, harvest_values)
+          build_nested_hash(result, path, value)
+        end
+
+        result
+      end
+
+      def self.build_nested_hash(current, path, value)
+        path.each_with_index do |key, index|
+          key = convert_key(key)
+
+          if index == path.length - 1
+            current[key] = value
+          else
+            next_key = convert_key(path[index + 1])
+            current[key] ||= next_key.is_a?(Integer) ? [] : {}
+            current = current[key]
+          end
+        end
+      end
+
+      def self.convert_key(key)
+        key.match?(/^\d+$/) ? key.to_i : key
+      end
+
+      def self.fetch_value(hash, harvest_values)
+        if hash["value_type"] == "static"
+          cast_value(hash["value"], hash["type"])
+        else
+          cast_value(harvest_values[hash["value"]], hash["type"])
+        end
+      end
+
+      def self.cast_value(value, type)
+        case type
+        when "string"
+          value.to_s
+        when "number"
+          value.to_i
+        when "boolean"
+          value == "true"
+        else
+          value
+        end
+      end
+    end
+  end
+end
diff --git a/server/lib/utils/payload_generator/dynamic_sql.rb b/server/lib/utils/payload_generator/dynamic_sql.rb
new file mode 100644
index 00000000..f2f02528
--- /dev/null
+++ b/server/lib/utils/payload_generator/dynamic_sql.rb
@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module Utils
+  module PayloadGenerator
+    module DynamicSql
+      def self.generate_query(model, harvest_values)
+        dynamic_query = model.query
+        input_config = model.json_schema["input"]
+
+        input_config.each do |config|
+          dynamic_var_name = ":#{config['name']}"
+          dynamic_var_value = config["value_type"] == "static" ? config["value"] : harvest_values[config["name"]]
+
+          dynamic_query.gsub!(dynamic_var_name, dynamic_var_value)
+        end
+
+        dynamic_query
+      end
+    end
+  end
+end
diff --git a/server/spec/lib/utils/payload_generator/ai_ml_spec.rb b/server/spec/lib/utils/payload_generator/ai_ml_spec.rb
new file mode 100644
index 00000000..a63d3bbe
--- /dev/null
+++ b/server/spec/lib/utils/payload_generator/ai_ml_spec.rb
@@ -0,0 +1,127 @@
+# frozen_string_literal: true
+
+require "rails_helper"
+
+RSpec.describe Utils::PayloadGenerator::AiMl do
+  describe ".generate_payload" do
+    context "when generating a nested payload from a complex flat hash" do
+      it "creates a valid nested structure with dynamic and static values" do
+        flat_hash = [
+          { "name" => "messages.0.role", "type" => "string", "value" => "key1", "value_type" => "dynamic" },
+          { "name" => "messages.0.content", "type" => "string", "value" => "key2", "value_type" => "dynamic" },
+          { "name" => "messages.0.test", "type" => "string", "value" => "hai", "value_type" => "static" },
+          { "name" => "messages.0.new.id", "type" => "number", "value" => "1", "value_type" => "static" },
+          { "name" => "messages.1.role", "type" => "string", "value" => "key3", "value_type" => "dynamic" },
+          { "name" => "messages.1.content", "type" => "string", "value" => "key4", "value_type" => "dynamic" },
+          { "name" => "messages.1.test", "type" => "string", "value" => "hello", "value_type" => "static" },
+          { "name" => "messages.1.details.info", "type" => "string", "value" => "key5", "value_type" => "dynamic" },
+          { "name" => "meta.version", "type" => "number", "value" => "3", "value_type" => "static" },
+          { "name" => "meta.status", "type" => "boolean", "value" => "true", "value_type" => "static" }
+        ]
+
+        harvest_values = {
+          "key1" => "user",
+          "key2" => "Hello there",
+          "key3" => "assistant",
+          "key4" => "How can I assist?",
+          "key5" => "additional info"
+        }
+
+        expected_payload = {
+          "messages" => [
+            {
+              "role" => "user",
+              "content" => "Hello there",
+              "test" => "hai",
+              "new" => {
+                "id" => 1
+              }
+            },
+            {
+              "role" => "assistant",
+              "content" => "How can I assist?",
+              "test" => "hello",
+              "details" => {
+                "info" => "additional info"
+              }
+            }
+          ],
+          "meta" => {
+            "version" => 3,
+            "status" => true
+          }
+        }
+
+        result = described_class.generate_payload(flat_hash, harvest_values)
+        expect(result).to eq(expected_payload)
+      end
+    end
+  end
+
+  context "when the input is empty" do
+    it "returns an empty hash" do
+      flat_hash = []
+      harvest_values = {}
+
+      expected_payload = {}
+
+      result = described_class.generate_payload(flat_hash, harvest_values)
+      expect(result).to eq(expected_payload)
+    end
+  end
+
+  context "when harvest_values are missing for dynamic values" do
+    it "handles missing dynamic values and returns the payload with nil values" do
+      flat_hash = [
+        { "name" => "messages.0.role", "type" => "string", "value" => "key1", "value_type" => "dynamic" },
+        { "name" => "messages.0.content", "type" => "string", "value" => "key2", "value_type" => "dynamic" },
+        { "name" => "messages.0.test", "type" => "string", "value" => "hai", "value_type" => "static" }
+      ]
+
+      harvest_values = {
+        # Dynamic values for key1 and key2 are missing
+      }
+
+      expected_payload = {
+        "messages" => [
+          {
+            "role" => "",
+            "content" => "",
+            "test" => "hai"
+          }
+        ]
+      }
+
+      result = described_class.generate_payload(flat_hash, harvest_values)
+      expect(result).to eq(expected_payload)
+    end
+  end
+
+  context "when invalid types are provided in flat_hash" do
+    it "returns the payload and skips invalid types" do
+      flat_hash = [
+        { "name" => "messages.0.role", "type" => "invalid_type", "value" => "key1", "value_type" => "dynamic" },
+        { "name" => "messages.0.content", "type" => "string", "value" => "key2", "value_type" => "dynamic" },
+        { "name" => "messages.0.test", "type" => "string", "value" => "hai", "value_type" => "static" }
+      ]
+
+      harvest_values = {
+        "key1" => "user",
+        "key2" => "Hello there"
+      }
+
+      expected_payload = {
+        "messages" => [
+          {
+            "role" => "user",
+            "content" => "Hello there",
+            "test" => "hai"
+          }
+        ]
+      }
+
+      result = described_class.generate_payload(flat_hash, harvest_values)
+      expect(result).to eq(expected_payload)
+    end
+  end
+end
diff --git a/server/spec/lib/utils/payload_generator/dynamic_sql_spec.rb b/server/spec/lib/utils/payload_generator/dynamic_sql_spec.rb
new file mode 100644
index 00000000..dcb5eef0
--- /dev/null
+++ b/server/spec/lib/utils/payload_generator/dynamic_sql_spec.rb
@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+require "rails_helper"
+
+RSpec.describe Utils::PayloadGenerator::DynamicSql do
+  describe ".generate_query" do
+    let!(:dynamic_sql_connector) do
+      create(:connector, connector_name: "Postgres", connector_type: "source", connector_category: "Dynamic SQL")
+    end
+
+    let!(:dynamic_sql_model) do
+      create(:model, query_type: :dynamic_sql, connector: dynamic_sql_connector,
+                     configuration: {
+                       json_schema: {
+                         input: [{ "name" => "name",
+                                   "type" => "string", "value" => "", "value_type" => "dynamic" },
+                                 { "name" => "age",
+                                   "type" => "number", "value" => "22", "value_type" => "static" },
+                                 { "name" => "gender",
+                                   "type" => "string", "value" => "", "value_type" => "dynamic" }],
+                         output: []
+                       },
+                       harvesters: []
+                     },
+                     query: "SELECT * FROM public.actor WHERE name=':name' AND age=:age AND gender=':gender'")
+    end
+
+    let(:harvesters) do
+      { "name" => "first_name", "gender" => "female" }
+    end
+
+    context "when correct input and harvest values are provided" do
+      it "replaces dynamic query values and return raw query" do
+        expected_query = "SELECT * FROM public.actor WHERE name='first_name' AND age=22 AND gender='female'"
+        generated_query = described_class.generate_query(dynamic_sql_model, harvesters)
+        expect(generated_query).to eq(expected_query)
+      end
+    end
+  end
+end

From d5e43d862cc45c3dbdc10ba6fed85e5a7165ecf9 Mon Sep 17 00:00:00 2001
From: "github-actions[bot]"
 <41898282+github-actions[bot]@users.noreply.github.com>
Date: Mon, 18 Nov 2024 10:27:30 -0500
Subject: [PATCH 43/48] chore(CE): Add session id to filter parameter (#591)
 (#475)

Co-authored-by: TivonB-AI2
---
 server/config/initializers/filter_parameter_logging.rb | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/server/config/initializers/filter_parameter_logging.rb b/server/config/initializers/filter_parameter_logging.rb
index 240bab25..84c58f6d 100644
--- a/server/config/initializers/filter_parameter_logging.rb
+++ b/server/config/initializers/filter_parameter_logging.rb
@@ -26,5 +26,6 @@
   :pin,                     
   :current_password,        
   :new_password,            
-  :ssn_last4,               
+  :ssn_last4,    
+  :session_id,           
 ]

From c797034063c8cb6f065ede8340cca57f2c43aabb Mon Sep 17 00:00:00 2001
From: "github-actions[bot]"
 <41898282+github-actions[bot]@users.noreply.github.com>
Date: Mon, 18 Nov 2024 10:28:13 -0500
Subject: [PATCH 44/48] chore(CE): Change title of AIS Data Store (#585) (#477)

Co-authored-by: TivonB-AI2
---
 integrations/Gemfile.lock                     |  2 +-
 .../destination/ais_data_store/client.rb      |  8 ++++----
 .../ais_data_store/config/meta.json           |  2 +-
 .../ais_data_store/config/spec.json           | 20 +++++++++----------
 .../lib/multiwoven/integrations/rollout.rb    |  2 +-
 .../destination/ais_data_store/client_spec.rb | 12 +++++------
 6 files changed, 23 insertions(+), 23 deletions(-)

diff --git a/integrations/Gemfile.lock b/integrations/Gemfile.lock
index 5a5979a7..aaa45bbe 100644
--- a/integrations/Gemfile.lock
+++ b/integrations/Gemfile.lock
@@ -7,7 +7,7 @@ GIT
 PATH
   remote: .
   specs:
-    multiwoven-integrations (0.15.2)
+    multiwoven-integrations (0.15.4)
       MailchimpMarketing
       activesupport
       async-websocket
diff --git a/integrations/lib/multiwoven/integrations/destination/ais_data_store/client.rb b/integrations/lib/multiwoven/integrations/destination/ais_data_store/client.rb
index 67d878d6..0b5ae33b 100644
--- a/integrations/lib/multiwoven/integrations/destination/ais_data_store/client.rb
+++ b/integrations/lib/multiwoven/integrations/destination/ais_data_store/client.rb
@@ -35,7 +35,7 @@ def discover(connection_config)
         catalog.to_multiwoven_message
       rescue StandardError => e
         handle_exception(e, {
-                           context: "POSTGRESQL:DISCOVER:EXCEPTION",
+                           context: "AIS:DATA:STORE:DISCOVER:EXCEPTION",
                            type: "error"
                          })
       ensure
@@ -54,14 +54,14 @@ def write(sync_config, records, action = "destination_insert")
 
         records.each do |record|
           query = Multiwoven::Integrations::Core::QueryBuilder.perform(action, table_name, record, primary_key)
-          logger.debug("POSTGRESQL:WRITE:QUERY query = #{query} sync_id = #{sync_config.sync_id} sync_run_id = #{sync_config.sync_run_id}")
+          logger.debug("AIS:DATA:STORE:WRITE:QUERY query = #{query} sync_id = #{sync_config.sync_id} sync_run_id = #{sync_config.sync_run_id}")
           begin
             response = db.exec(query)
             write_success += 1
             log_message_array << log_request_response("info", query, response)
           rescue StandardError => e
             handle_exception(e, {
-                               context: "POSTGRESQL:RECORD:WRITE:EXCEPTION",
+                               context: "AIS:DATA:STORE:RECORD:WRITE:EXCEPTION",
                                type: "error",
                                sync_id: sync_config.sync_id,
                                sync_run_id: sync_config.sync_run_id
@@ -73,7 +73,7 @@ def write(sync_config, records, action = "destination_insert")
         tracking_message(write_success, write_failure, log_message_array)
       rescue StandardError => e
         handle_exception(e, {
-                           context: "POSTGRESQL:RECORD:WRITE:EXCEPTION",
+                           context: "AIS:DATA:STORE:RECORD:WRITE:EXCEPTION",
                            type: "error",
                            sync_id: sync_config.sync_id,
                            sync_run_id: sync_config.sync_run_id
diff --git a/integrations/lib/multiwoven/integrations/destination/ais_data_store/config/meta.json b/integrations/lib/multiwoven/integrations/destination/ais_data_store/config/meta.json
index 194991f1..1fd002ff 100644
--- a/integrations/lib/multiwoven/integrations/destination/ais_data_store/config/meta.json
+++ b/integrations/lib/multiwoven/integrations/destination/ais_data_store/config/meta.json
@@ -5,7 +5,7 @@
     "connector_type": "destination",
     "category": "Database",
     "documentation_url": "https://docs.mutliwoven.com",
-    "github_issue_label": "destination-postgresql",
+    "github_issue_label": "destination-ais-data-store",
     "icon": "icon.svg",
     "license": "MIT",
     "release_stage": "alpha",
diff --git a/integrations/lib/multiwoven/integrations/destination/ais_data_store/config/spec.json b/integrations/lib/multiwoven/integrations/destination/ais_data_store/config/spec.json
index 7ce099e6..fcf370e4 100644
--- a/integrations/lib/multiwoven/integrations/destination/ais_data_store/config/spec.json
+++ b/integrations/lib/multiwoven/integrations/destination/ais_data_store/config/spec.json
@@ -1,9 +1,9 @@
 {
-  "documentation_url": "https://docs.multiwoven.com/integrations/sources/postgresql",
+  "documentation_url": "https://docs.multiwoven.com/integrations/destination/ais-data-store",
   "stream_type": "dynamic",
   "connection_specification": {
     "$schema": "http://json-schema.org/draft-07/schema#",
-    "title": "Postgresql",
+    "title": "AIS Data Store",
     "type": "object",
     "required": ["host", "port", "database", "schema"],
     "properties": {
@@ -19,8 +19,8 @@
             "readOnly": true
           },
           "username": {
-            "description": "Username refers to your individual PostgreSQL login credentials. At a minimum, the user associated with these credentials must be granted read access to the data intended for synchronization.",
-            "examples": ["POSTGRESQL_USER"],
+            "description": "Username refers to your individual AIS Data Store login credentials. At a minimum, the user associated with these credentials must be granted read access to the data intended for synchronization.",
+            "examples": ["AIS_DATA_STORE_USER"],
             "type": "string",
             "title": "Username",
             "order": 1
@@ -36,29 +36,29 @@
         "order": 0
       },
       "host": {
-        "description": "The hostname or IP address of your PostgreSQL server.",
+        "description": "The hostname or IP address of your AIS Data Store server.",
         "examples": ["127.0.0.1"],
         "type": "string",
         "title": "Host",
         "order": 1
       },
       "port": {
-        "description": "The port number for your PostgreSQL server, which defaults to 5432, may vary based on your configuration. ",
+        "description": "The port number for your AIS Data Store server, which defaults to 5432, may vary based on your configuration. ",
         "examples": ["5432"],
         "type": "string",
         "title": "Port",
         "order": 2
       },
       "database": {
-        "description": "The specific PostgreSQL database to connect to.",
-        "examples": ["POSTGRESQL_DB"],
+        "description": "The specific AIS Data Store database to connect to.",
+        "examples": ["AIS_DATA_STORE_DB"],
         "type": "string",
         "title": "Database",
         "order": 3
       },
       "schema": {
-        "description": "The schema within the PostgreSQL database.",
-        "examples": ["POSTGRESQL_SCHEMA"],
+        "description": "The schema within the AIS Data Store database.",
+        "examples": ["AIS_DATA_STORE_SCHEMA"],
         "type": "string",
         "title": "Schema",
         "order": 4
diff --git a/integrations/lib/multiwoven/integrations/rollout.rb b/integrations/lib/multiwoven/integrations/rollout.rb
index 3c69f4f9..85268d83 100644
--- a/integrations/lib/multiwoven/integrations/rollout.rb
+++ b/integrations/lib/multiwoven/integrations/rollout.rb
@@ -2,7 +2,7 @@
 
 module Multiwoven
   module Integrations
-    VERSION = "0.15.3"
+    VERSION = "0.15.4"
 
     ENABLED_SOURCES = %w[
       Snowflake
diff --git a/integrations/spec/multiwoven/integrations/destination/ais_data_store/client_spec.rb b/integrations/spec/multiwoven/integrations/destination/ais_data_store/client_spec.rb
index 8419c92f..662a4938 100644
--- a/integrations/spec/multiwoven/integrations/destination/ais_data_store/client_spec.rb
+++ b/integrations/spec/multiwoven/integrations/destination/ais_data_store/client_spec.rb
@@ -5,13 +5,13 @@
   let(:sync_config) do
     {
       "source": {
-        "name": "PostgresqlSourceConnector",
+        "name": "AISDataStoreSourceConnector",
         "type": "source",
         "connection_specification": {
           "credentials": {
             "auth_type": "username/password",
-            "username": ENV["POSTGRESQL_USERNAME"],
-            "password": ENV["POSTGRESQL_PASSWORD"]
+            "username": ENV["AISDATASTORE_USERNAME"],
+            "password": ENV["AISDATASTORE_PASSWORD"]
           },
           "host": "test.pg.com",
           "port": "8080",
@@ -25,8 +25,8 @@
         "connection_specification": {
           "credentials": {
             "auth_type": "username/password",
-            "username": ENV["POSTGRESQL_USERNAME"],
-            "password": ENV["POSTGRESQL_PASSWORD"]
+            "username": ENV["AISDATASTORE_USERNAME"],
+            "password": ENV["AISDATASTORE_PASSWORD"]
           },
           "host": "test.pg.com",
           "port": "8080",
@@ -192,7 +192,7 @@
       allow(client).to receive(:create_connection).and_raise(StandardError.new("test error"))
       expect(client).to receive(:handle_exception).with(
         an_instance_of(StandardError), {
-          context: "POSTGRESQL:DISCOVER:EXCEPTION",
+          context: "AIS:DATA:STORE:DISCOVER:EXCEPTION",
           type: "error"
         }
       )

From cf716da08615e0c1166bfec8719e6b2fd1b21e4b Mon Sep 17 00:00:00 2001
From: "Rafael E. O'Neill"
 <106079170+RafaelOAiSquared@users.noreply.github.com>
Date: Mon, 18 Nov 2024 15:02:43 -0400
Subject: [PATCH 45/48] feat(CE): on_failure report error to AppSignal (#478)

* feat(CE): on_failure report error to AppSignal

* feat(EE): added on_failure to deployment workflows

---------

Co-authored-by: developer-united <developer@multiwoven.com>
---
 .github/workflows/deploy-server.yml    | 19 +++++++++++++++++++
 .github/workflows/deploy-ui.yml        | 18 ++++++++++++++++++
 .github/workflows/release.yaml         | 19 +++++++++++++++++++
 .github/workflows/sync-ce-commits.yaml | 19 +++++++++++++++++++
 4 files changed, 75 insertions(+)

diff --git a/.github/workflows/deploy-server.yml b/.github/workflows/deploy-server.yml
index 913d15e4..5b7b7208 100644
--- a/.github/workflows/deploy-server.yml
+++ b/.github/workflows/deploy-server.yml
@@ -54,3 +54,22 @@ jobs:
             --set multiwovenServer.multiwovenServer.image.tag=$IMAGE_TAG \
             --set multiwovenWorker.multiwovenWorker.image.repository=$IMAGE_NAME \
             --set multiwovenWorker.multiwovenWorker.image.tag=$IMAGE_TAG
+
+  on_failure:
+    environment: community
+    if: ${{ failure() }}
+    needs: [ build_and_push, deploy ]
+    runs-on: ubuntu-latest
+    steps:
+      - name: Report error to AppSignal
+        env:
+          ACTION: Release
+          APPSIGNAL_PUSH_API_KEY: ${{ secrets.APPSIGNAL_PUSH_API_KEY }}
+          MESSAGE: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}
+          NAME: Deploy Server to Community failed
+        run: |
+          EPOCH=$(date +%s)
+          curl -verbose -H \"Content-Type: application/json\" \
+            -d "{ \"action\": \"$ACTION\", \"timestamp\": $EPOCH, \"namespace\": \"github\", \"error\": { \"name\": $NAME, \"message\": $MESSAGE } }" \
+            -X POST \
+            https://appsignal-endpoint.net/errors?api_key=$APPSIGNAL_PUSH_API_KEY
\ No newline at end of file
diff --git a/.github/workflows/deploy-ui.yml b/.github/workflows/deploy-ui.yml
index cadf51bd..5c1b6322 100644
--- a/.github/workflows/deploy-ui.yml
+++ b/.github/workflows/deploy-ui.yml
@@ -54,3 +54,21 @@ jobs:
             --set multiwovenUI.multiwovenUI.image.tag=$IMAGE_TAG
           kubectl rollout restart deployment/multiwoven-server -n multiwoven
       
+  on_failure:
+    environment: community
+    if: ${{ failure() }}
+    needs: [ build_and_push, deploy ]
+    runs-on: ubuntu-latest
+    steps:
+      - name: Report error to AppSignal
+        env:
+          ACTION: Deployment
+          APPSIGNAL_PUSH_API_KEY: ${{ secrets.APPSIGNAL_PUSH_API_KEY }}
+          MESSAGE: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}
+          NAME: Deploy UI to Community failed
+        run: |
+          EPOCH=$(date +%s)
+          curl -verbose -H \"Content-Type: application/json\" \
+            -d "{ \"action\": \"$ACTION\", \"timestamp\": $EPOCH, \"namespace\": \"github\", \"error\": { \"name\": $NAME, \"message\": $MESSAGE } }" \
+            -X POST \
+            https://appsignal-endpoint.net/errors?api_key=$APPSIGNAL_PUSH_API_KEY
\ No newline at end of file
diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml
index 9b3d5338..76ff7bb9 100644
--- a/.github/workflows/release.yaml
+++ b/.github/workflows/release.yaml
@@ -89,3 +89,22 @@ jobs:
           source_branch: "release-changes-${{ steps.versioning.outputs.new_tag }}"
           destination_branch: "main"
           pr_label: "release,auto-pr"
+
+  on_failure:
+    environment: community
+    if: ${{ failure() }}
+    needs: [ prepare-changelog ]
+    runs-on: ubuntu-latest
+    steps:
+      - name: Report error to AppSignal
+        env:
+          ACTION: Release
+          APPSIGNAL_PUSH_API_KEY: ${{ secrets.APPSIGNAL_PUSH_API_KEY }}
+          MESSAGE: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}
+          NAME: Community Release failed
+        run: |
+          EPOCH=$(date +%s)
+          curl -verbose -H \"Content-Type: application/json\" \
+            -d "{ \"action\": \"$ACTION\", \"timestamp\": $EPOCH, \"namespace\": \"github\", \"error\": { \"name\": $NAME, \"message\": $MESSAGE } }" \
+            -X POST \
+            https://appsignal-endpoint.net/errors?api_key=$APPSIGNAL_PUSH_API_KEY
\ No newline at end of file
diff --git a/.github/workflows/sync-ce-commits.yaml b/.github/workflows/sync-ce-commits.yaml
index f1241c37..19b140d8 100644
--- a/.github/workflows/sync-ce-commits.yaml
+++ b/.github/workflows/sync-ce-commits.yaml
@@ -70,3 +70,22 @@ jobs:
             git checkout main
             git branch -D $BRANCH_NAME
           done
+
+  on_failure:
+    environment: staging
+    if: ${{ failure() }}
+    needs: [ cherry-pick-ce-commits ]
+    runs-on: ubuntu-latest
+    steps:
+      - name: Report error to AppSignal
+        env:
+          ACTION: Sync Commits
+          APPSIGNAL_PUSH_API_KEY: ${{ secrets.APPSIGNAL_PUSH_API_KEY }}
+          MESSAGE: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}
+          NAME: Sync Commits from EE to CE failed
+        run: |
+          EPOCH=$(date +%s)
+          curl -verbose -H \"Content-Type: application/json\" \
+            -d "{ \"action\": \"$ACTION\", \"timestamp\": $EPOCH, \"namespace\": \"github\", \"error\": { \"name\": $NAME, \"message\": $MESSAGE } }" \
+            -X POST \
+            https://appsignal-endpoint.net/errors?api_key=$APPSIGNAL_PUSH_API_KEY
\ No newline at end of file

From d3ce365c26c59f0e9981c8b273123ab8492a399f Mon Sep 17 00:00:00 2001
From: "Rafael E. O'Neill"
 <106079170+RafaelOAiSquared@users.noreply.github.com>
Date: Tue, 19 Nov 2024 04:05:22 -0400
Subject: [PATCH 46/48] Multiwoven release v0.34.0 (#480)

Co-authored-by: github-actions <developer@multiwoven.com>
---
 release-notes.md | 27 ++++++++++++++++++++++++---
 1 file changed, 24 insertions(+), 3 deletions(-)

diff --git a/release-notes.md b/release-notes.md
index 7f136c91..316476ef 100644
--- a/release-notes.md
+++ b/release-notes.md
@@ -2,14 +2,35 @@
 
 All notable changes to this project will be documented in this file.
 
-## [0.33.0] - 2024-11-11
+## [0.34.0] - 2024-11-18
 
 ### 🚀 Features
 
-- *(CE)* Added Mailchimp Destination Connector (#437)
+- *(CE)* Add HTTP Model Source Connector (#466)
+- *(CE)* AIS Data store destination connector (#462)
+- *(CE)* Added dynamic_sql type to model (#576)
+- *(CE)* Fetch json_schema from model (#581)
+- *(CE)* Fetch data for dsql models (#596) (#476)
+- *(CE)* On_failure report error to AppSignal (#478)
+
+### 🐛 Bug Fixes
+
+- *(CE)* Data app deletion error fix
+- *(CE)* Mailchimp dependency build issue
+- *(CE)* Http model config (#464)
+- *(CE)* Ais data store icon (#467)
+- *(CE)* Dockerfile update free tds (#470)
 
 ### ⚙️ Miscellaneous Tasks
 
-- *(CE)* Add additional_remarks in feedback model (#454)
+- *(CE)* Add icons for ais data store and http model (#460)
+- *(CE)* Add timeout to http client (#458)
+- *(CE)* Dynamic sql model type added (#465)
+- *(CE)* Update integration gem version (#457)
+- *(CE)* Change http model name (#469)
+- *(CE)* Update HTTP Model spec (#583) (#474)
+- *(CE)* Server Gem Update 0.15.3 (#589) (#471)
+- *(CE)* Add session id to filter parameter (#591) (#475)
+- *(CE)* Change title of AIS Data Store (#585) (#477)
 
 <!-- generated by git-cliff -->

From c584c3808e349d869713b7ebfd318324e8618700 Mon Sep 17 00:00:00 2001
From: Tushar Selvakumar <54372016+macintushar@users.noreply.github.com>
Date: Tue, 19 Nov 2024 16:11:10 +0530
Subject: [PATCH 47/48] fix(CE): fixed build error

---
 ui/src/services/models.ts | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/ui/src/services/models.ts b/ui/src/services/models.ts
index d323104b..33c6702c 100644
--- a/ui/src/services/models.ts
+++ b/ui/src/services/models.ts
@@ -58,8 +58,10 @@ export const getModelPreview = async (query: string, connector_id: string): Prom
   return apiRequest(url, { query: query });
 };
 
-export const getAllModels = async ({ type = 'data' }: GetAllModelsProps): Promise<APIData> =>
-  multiwovenFetch<null, APIData>({
+export const getAllModels = async ({
+  type = 'data',
+}: GetAllModelsProps): Promise<ApiResponse<GetAllModelsResponse[]>> =>
+  multiwovenFetch<null, ApiResponse<GetAllModelsResponse[]>>({
     method: 'get',
     url: type ? `/models?query_type=${type}` : '/models',
   });

From 17cd11dd6778d76ac201d15388fc4eaead92f7ba Mon Sep 17 00:00:00 2001
From: "Rafael E. O'Neill"
 <106079170+RafaelOAiSquared@users.noreply.github.com>
Date: Tue, 19 Nov 2024 08:32:32 -0400
Subject: [PATCH 48/48] fix(CE): removed reference to build_and_push job

Co-authored-by: developer-united <developer@multiwoven.com>
---
 .github/workflows/deploy-server.yml | 2 +-
 .github/workflows/deploy-ui.yml     | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/deploy-server.yml b/.github/workflows/deploy-server.yml
index 5b7b7208..66dd9942 100644
--- a/.github/workflows/deploy-server.yml
+++ b/.github/workflows/deploy-server.yml
@@ -58,7 +58,7 @@ jobs:
   on_failure:
     environment: community
     if: ${{ failure() }}
-    needs: [ build_and_push, deploy ]
+    needs: [ deploy ]
     runs-on: ubuntu-latest
     steps:
       - name: Report error to AppSignal
diff --git a/.github/workflows/deploy-ui.yml b/.github/workflows/deploy-ui.yml
index 5c1b6322..49babaf7 100644
--- a/.github/workflows/deploy-ui.yml
+++ b/.github/workflows/deploy-ui.yml
@@ -57,7 +57,7 @@ jobs:
   on_failure:
     environment: community
     if: ${{ failure() }}
-    needs: [ build_and_push, deploy ]
+    needs: [ deploy ]
     runs-on: ubuntu-latest
     steps:
       - name: Report error to AppSignal