Connector: Move models in file per connector (#2593)

dust-tt · Nov 23, 2023 · 59d267c · 59d267c
1 parent 6af0e00
commit 59d267c
Show file tree

Hide file tree

Showing 35 changed files with 1,417 additions and 1,434 deletions.
diff --git a/connectors/migrations/20230505_add_notion_connector_state.ts b/connectors/migrations/20230505_add_notion_connector_state.ts
@@ -1,4 +1,5 @@
-import { Connector, NotionConnectorState } from "@connectors/lib/models";
+import { Connector } from "@connectors/lib/models";
+import { NotionConnectorState } from "@connectors/lib/models/notion";
 
 async function main() {
   const connectors = await Connector.findAll();

diff --git a/connectors/migrations/20230725_slack_channel_permissions.ts b/connectors/migrations/20230725_slack_channel_permissions.ts
@@ -1,5 +1,6 @@
 import { getChannels } from "@connectors/connectors/slack/temporal/activities";
-import { Connector, SlackChannel } from "@connectors/lib/models";
+import { Connector } from "@connectors/lib/models";
+import { SlackChannel } from "@connectors/lib/models/slack";
 
 async function main() {
   const slackConnectors = await Connector.findAll({

diff --git a/connectors/migrations/20230828_notion_block_parents.ts b/connectors/migrations/20230828_notion_block_parents.ts
@@ -7,7 +7,8 @@ import PQueue from "p-queue";
 import { Op } from "sequelize";
 
 import { getBlockParentMemoized } from "@connectors/connectors/notion/lib/notion_api";
-import { Connector, NotionDatabase, NotionPage } from "@connectors/lib/models";
+import { Connector } from "@connectors/lib/models";
+import { NotionDatabase, NotionPage } from "@connectors/lib/models/notion";
 import { nango_client } from "@connectors/lib/nango_client";
 import mainLogger from "@connectors/logger/logger";
 

diff --git a/connectors/migrations/20230906_2_slack_fill_parents_field.ts b/connectors/migrations/20230906_2_slack_fill_parents_field.ts
@@ -2,7 +2,8 @@ import { existsSync, readFileSync, writeFileSync } from "fs";
 import { Op } from "sequelize";
 
 import { updateDocumentParentsField } from "@connectors/lib/data_sources";
-import { Connector, SlackMessages } from "@connectors/lib/models";
+import { Connector } from "@connectors/lib/models";
+import { SlackMessages } from "@connectors/lib/models/slack";
 
 async function main() {
   if (!process.argv[2]) {

diff --git a/connectors/migrations/20230906_3_github_fill_parents_field.ts b/connectors/migrations/20230906_3_github_fill_parents_field.ts
@@ -6,11 +6,8 @@ import {
   getIssueDocumentId,
 } from "@connectors/connectors/github/temporal/activities";
 import { updateDocumentParentsField } from "@connectors/lib/data_sources";
-import {
-  Connector,
-  GithubDiscussion,
-  GithubIssue,
-} from "@connectors/lib/models";
+import { Connector } from "@connectors/lib/models";
+import { GithubDiscussion, GithubIssue } from "@connectors/lib/models/github";
 
 async function main() {
   if (!process.argv[2]) {

diff --git a/connectors/migrations/20230906_notion_fill_parents_field.ts b/connectors/migrations/20230906_notion_fill_parents_field.ts
@@ -2,7 +2,8 @@ import { existsSync, readFileSync, writeFileSync } from "fs";
 import { Op } from "sequelize";
 
 import { updateAllParentsFields } from "@connectors/connectors/notion/lib/parents";
-import { Connector, NotionDatabase, NotionPage } from "@connectors/lib/models";
+import { Connector } from "@connectors/lib/models";
+import { NotionDatabase, NotionPage } from "@connectors/lib/models/notion";
 
 async function main() {
   if (!process.argv[2]) {

diff --git a/connectors/migrations/20231109_2_create_gdrive_config.ts b/connectors/migrations/20231109_2_create_gdrive_config.ts
@@ -1,4 +1,5 @@
-import { Connector, GoogleDriveConfig } from "@connectors/lib/models";
+import { Connector } from "@connectors/lib/models";
+import { GoogleDriveConfig } from "@connectors/lib/models/google_drive";
 
 async function main() {
   const gDriveConnectors = await Connector.findAll({

diff --git a/connectors/migrations/20231109_incident_gdrive_non_deleted_files.ts b/connectors/migrations/20231109_incident_gdrive_non_deleted_files.ts
@@ -2,7 +2,8 @@ import { Sequelize } from "sequelize";
 
 import { dataSourceConfigFromConnector } from "@connectors/lib/api/data_source_config";
 import { deleteFromDataSource } from "@connectors/lib/data_sources";
-import { Connector, GoogleDriveFiles } from "@connectors/lib/models";
+import { Connector } from "@connectors/lib/models";
+import { GoogleDriveFiles } from "@connectors/lib/models/google_drive";
 
 // To be run from connectors with `CORE_DATABASE_URI` and `FRONT_DATABASE_URI` set.
 const { CORE_DATABASE_URI, FRONT_DATABASE_URI, LIVE = false } = process.env;

diff --git a/connectors/src/admin/cli.ts b/connectors/src/admin/cli.ts
@@ -14,12 +14,9 @@ import {
 } from "@connectors/connectors/google_drive/temporal/client";
 import { toggleSlackbot } from "@connectors/connectors/slack/bot";
 import { launchSlackSyncOneThreadWorkflow } from "@connectors/connectors/slack/temporal/client";
-import {
-  Connector,
-  GoogleDriveFiles,
-  NotionDatabase,
-  NotionPage,
-} from "@connectors/lib/models";
+import { Connector } from "@connectors/lib/models";
+import { GoogleDriveFiles } from "@connectors/lib/models/google_drive";
+import { NotionDatabase, NotionPage } from "@connectors/lib/models/notion";
 import { Result } from "@connectors/lib/result";
 
 const connectors = async (command: string, args: parseArgs.ParsedArgs) => {

diff --git a/connectors/src/admin/db.ts b/connectors/src/admin/db.ts
@@ -1,31 +1,36 @@
 import { Sequelize } from "sequelize";
 
+import { Connector, sequelize_conn } from "@connectors/lib/models";
 import {
-  Connector,
   GithubConnectorState,
   GithubDiscussion,
   GithubIssue,
+} from "@connectors/lib/models/github";
+import {
   GoogleDriveConfig,
   GoogleDriveFiles,
   GoogleDriveFolders,
   GoogleDriveSyncToken,
   GoogleDriveWebhook,
+} from "@connectors/lib/models/google_drive";
+import {
+  IntercomArticle,
+  IntercomCollection,
+} from "@connectors/lib/models/intercom";
+import {
   NotionConnectorBlockCacheEntry,
   NotionConnectorPageCacheEntry,
   NotionConnectorResourcesToCheckCacheEntry,
   NotionConnectorState,
   NotionDatabase,
   NotionPage,
-  sequelize_conn,
+} from "@connectors/lib/models/notion";
+import {
   SlackChannel,
   SlackChatBotMessage,
   SlackConfiguration,
   SlackMessages,
-} from "@connectors/lib/models";
-import {
-  IntercomArticle,
-  IntercomCollection,
-} from "@connectors/lib/models/intercom";
+} from "@connectors/lib/models/slack";
 import logger from "@connectors/logger/logger";
 
 async function main(): Promise<void> {

diff --git a/connectors/src/api/get_connector.ts b/connectors/src/api/get_connector.ts
@@ -1,11 +1,8 @@
 import { Request, Response } from "express";
 
-import {
-  Connector,
-  GithubDiscussion,
-  GithubIssue,
-  NotionPage,
-} from "@connectors/lib/models";
+import { Connector } from "@connectors/lib/models";
+import { GithubDiscussion, GithubIssue } from "@connectors/lib/models/github";
+import { NotionPage } from "@connectors/lib/models/notion";
 import { apiError, withLogging } from "@connectors/logger/withlogging";
 import { ConnectorType } from "@connectors/types/connector";
 import { ConnectorsAPIErrorResponse } from "@connectors/types/errors";

diff --git a/connectors/src/api/slack_channels_linked_with_agent.ts b/connectors/src/api/slack_channels_linked_with_agent.ts
@@ -7,7 +7,8 @@ import { Op } from "sequelize";
 import { joinChannel } from "@connectors/connectors/slack/lib/channels";
 import { getChannels } from "@connectors/connectors/slack/temporal/activities";
 import { APIErrorWithStatusCode } from "@connectors/lib/error";
-import { sequelize_conn, SlackChannel } from "@connectors/lib/models";
+import { sequelize_conn } from "@connectors/lib/models";
+import { SlackChannel } from "@connectors/lib/models/slack";
 import { apiError, withLogging } from "@connectors/logger/withlogging";
 
 const PatchSlackChannelsLinkedWithAgentReqBodySchema = t.type({

diff --git a/connectors/src/api/webhooks/webhook_github.ts b/connectors/src/api/webhooks/webhook_github.ts
@@ -21,7 +21,8 @@ import {
   launchGithubReposSyncWorkflow,
 } from "@connectors/connectors/github/temporal/client";
 import { assertNever } from "@connectors/lib/assert_never";
-import { Connector, GithubConnectorState } from "@connectors/lib/models";
+import { Connector } from "@connectors/lib/models";
+import { GithubConnectorState } from "@connectors/lib/models/github";
 import mainLogger from "@connectors/logger/logger";
 import { withLogging } from "@connectors/logger/withlogging";
 import { ConnectorsAPIErrorResponse } from "@connectors/types/errors";

diff --git a/connectors/src/api/webhooks/webhook_google_drive.ts b/connectors/src/api/webhooks/webhook_google_drive.ts
@@ -2,7 +2,7 @@ import { Request, Response } from "express";
 
 import { launchGoogleDriveIncrementalSyncWorkflow } from "@connectors/connectors/google_drive/temporal/client";
 import { APIErrorWithStatusCode } from "@connectors/lib/error";
-import { GoogleDriveWebhook } from "@connectors/lib/models";
+import { GoogleDriveWebhook } from "@connectors/lib/models/google_drive";
 import { apiError, withLogging } from "@connectors/logger/withlogging";
 
 type GoogleDriveWebhookResBody = null | APIErrorWithStatusCode;

diff --git a/connectors/src/api/webhooks/webhook_slack.ts b/connectors/src/api/webhooks/webhook_slack.ts
@@ -8,11 +8,8 @@ import {
 } from "@connectors/connectors/slack/temporal/client";
 import { launchSlackGarbageCollectWorkflow } from "@connectors/connectors/slack/temporal/client";
 import { APIErrorWithStatusCode } from "@connectors/lib/error";
-import {
-  Connector,
-  SlackChannel,
-  SlackConfiguration,
-} from "@connectors/lib/models";
+import { Connector } from "@connectors/lib/models";
+import { SlackChannel, SlackConfiguration } from "@connectors/lib/models/slack";
 import { Ok } from "@connectors/lib/result";
 import logger from "@connectors/logger/logger";
 import { apiError, withLogging } from "@connectors/logger/withlogging";

diff --git a/connectors/src/connectors/github/index.ts b/connectors/src/connectors/github/index.ts
@@ -3,13 +3,11 @@ import {
   validateInstallationId,
 } from "@connectors/connectors/github/lib/github_api";
 import { launchGithubFullSyncWorkflow } from "@connectors/connectors/github/temporal/client";
+import { Connector, ModelId, sequelize_conn } from "@connectors/lib/models";
 import {
-  Connector,
   GithubConnectorState,
   GithubIssue,
-  ModelId,
-  sequelize_conn,
-} from "@connectors/lib/models";
+} from "@connectors/lib/models/github";
 import { Err, Ok, Result } from "@connectors/lib/result";
 import mainLogger from "@connectors/logger/logger";
 import { DataSourceConfig } from "@connectors/types/data_source_config";

diff --git a/connectors/src/connectors/github/temporal/activities.ts b/connectors/src/connectors/github/temporal/activities.ts
@@ -15,11 +15,8 @@ import {
   deleteFromDataSource,
   upsertToDatasource,
 } from "@connectors/lib/data_sources";
-import {
-  Connector,
-  GithubDiscussion,
-  GithubIssue,
-} from "@connectors/lib/models";
+import { Connector } from "@connectors/lib/models";
+import { GithubDiscussion, GithubIssue } from "@connectors/lib/models/github";
 import { syncStarted, syncSucceeded } from "@connectors/lib/sync_status";
 import mainLogger from "@connectors/logger/logger";
 import { DataSourceConfig } from "@connectors/types/data_source_config";

diff --git a/connectors/src/connectors/google_drive/index.ts b/connectors/src/connectors/google_drive/index.ts
@@ -7,16 +7,14 @@ import {
   registerWebhook,
 } from "@connectors/connectors/google_drive/lib";
 import { ConnectorPermissionRetriever } from "@connectors/connectors/interface";
+import { Connector, ModelId, sequelize_conn } from "@connectors/lib/models";
 import {
-  Connector,
   GoogleDriveConfig,
   GoogleDriveFiles,
   GoogleDriveFolders,
   GoogleDriveSyncToken,
   GoogleDriveWebhook,
-  ModelId,
-  sequelize_conn,
-} from "@connectors/lib/models.js";
+} from "@connectors/lib/models/google_drive";
 import { nangoDeleteConnection } from "@connectors/lib/nango_client";
 import { Err, Ok, type Result } from "@connectors/lib/result.js";
 import logger from "@connectors/logger/logger";

diff --git a/connectors/src/connectors/google_drive/lib.ts b/connectors/src/connectors/google_drive/lib.ts
@@ -2,7 +2,8 @@ import memoize from "lodash.memoize";
 import { v4 as uuidv4 } from "uuid";
 
 import { HTTPError } from "@connectors/lib/error";
-import { Connector, GoogleDriveFiles, ModelId } from "@connectors/lib/models";
+import { Connector, ModelId } from "@connectors/lib/models";
+import { GoogleDriveFiles } from "@connectors/lib/models/google_drive";
 import { Err, Ok, type Result } from "@connectors/lib/result.js";
 
 import { getAuthObject } from "./temporal/activities";

diff --git a/connectors/src/connectors/google_drive/temporal/activities.ts b/connectors/src/connectors/google_drive/temporal/activities.ts
@@ -24,16 +24,14 @@ import { CreationAttributes, literal, Op } from "sequelize";
 import { dataSourceConfigFromConnector } from "@connectors/lib/api/data_source_config";
 import { dpdf2text } from "@connectors/lib/dpdf2text";
 import { ExternalOauthTokenError } from "@connectors/lib/error";
+import { Connector, ModelId, sequelize_conn } from "@connectors/lib/models";
 import {
-  Connector,
   GoogleDriveConfig,
   GoogleDriveFiles,
   GoogleDriveFolders,
   GoogleDriveSyncToken,
   GoogleDriveWebhook,
-  ModelId,
-  sequelize_conn,
-} from "@connectors/lib/models";
+} from "@connectors/lib/models/google_drive";
 import logger from "@connectors/logger/logger";
 
 import { registerWebhook } from "../lib";

diff --git a/connectors/src/connectors/notion/index.ts b/connectors/src/connectors/notion/index.ts
@@ -6,14 +6,12 @@ import {
   stopNotionSyncWorkflow,
 } from "@connectors/connectors/notion/temporal/client";
 import { dataSourceConfigFromConnector } from "@connectors/lib/api/data_source_config";
+import { Connector, ModelId, sequelize_conn } from "@connectors/lib/models";
 import {
-  Connector,
-  ModelId,
   NotionConnectorState,
   NotionDatabase,
   NotionPage,
-  sequelize_conn,
-} from "@connectors/lib/models";
+} from "@connectors/lib/models/notion";
 import {
   nango_client,
   nangoDeleteConnection,

diff --git a/connectors/src/connectors/notion/lib/connectors_db_helpers.ts b/connectors/src/connectors/notion/lib/connectors_db_helpers.ts
@@ -1,9 +1,5 @@
-import {
-  Connector,
-  ModelId,
-  NotionDatabase,
-  NotionPage,
-} from "@connectors/lib/models";
+import { Connector, ModelId } from "@connectors/lib/models";
+import { NotionDatabase, NotionPage } from "@connectors/lib/models/notion";
 import { DataSourceInfo } from "@connectors/types/data_source_config";
 
 // Note: this function does not let you "remove" a skipReason.

diff --git a/connectors/src/connectors/notion/lib/parents.ts b/connectors/src/connectors/notion/lib/parents.ts
@@ -8,12 +8,8 @@ import {
   getPageChildrenOf,
 } from "@connectors/connectors/notion/lib/connectors_db_helpers";
 import { updateDocumentParentsField } from "@connectors/lib/data_sources";
-import {
-  Connector,
-  ModelId,
-  NotionDatabase,
-  NotionPage,
-} from "@connectors/lib/models";
+import { Connector, ModelId } from "@connectors/lib/models";
+import { NotionDatabase, NotionPage } from "@connectors/lib/models/notion";
 import logger from "@connectors/logger/logger";
 
 /** Compute the parents field for a notion pageOrDb See the [Design

diff --git a/connectors/src/connectors/notion/temporal/activities.ts b/connectors/src/connectors/notion/temporal/activities.ts
@@ -41,16 +41,15 @@ import {
   MAX_DOCUMENT_TXT_LEN,
   upsertToDatasource,
 } from "@connectors/lib/data_sources";
+import { Connector, ModelId } from "@connectors/lib/models";
 import {
-  Connector,
-  ModelId,
   NotionConnectorBlockCacheEntry,
   NotionConnectorPageCacheEntry,
   NotionConnectorResourcesToCheckCacheEntry,
   NotionConnectorState,
   NotionDatabase,
   NotionPage,
-} from "@connectors/lib/models";
+} from "@connectors/lib/models/notion";
 import { getAccessTokenFromNango } from "@connectors/lib/nango_helpers";
 import { syncStarted, syncSucceeded } from "@connectors/lib/sync_status";
 import mainLogger from "@connectors/logger/logger";

diff --git a/connectors/src/connectors/slack/bot.ts b/connectors/src/connectors/slack/bot.ts
@@ -15,13 +15,12 @@ import {
   RetrievalDocumentType,
   UserMessageType,
 } from "@connectors/lib/dust_api";
+import { Connector, ModelId } from "@connectors/lib/models";
 import {
-  Connector,
-  ModelId,
   SlackChannel,
   SlackChatBotMessage,
   SlackConfiguration,
-} from "@connectors/lib/models";
+} from "@connectors/lib/models/slack";
 import { Err, Ok, Result } from "@connectors/lib/result";
 import logger from "@connectors/logger/logger";
 

diff --git a/connectors/src/connectors/slack/index.ts b/connectors/src/connectors/slack/index.ts
@@ -11,14 +11,12 @@ import { getChannels } from "@connectors/connectors/slack//temporal/activities";
 import { joinChannel } from "@connectors/connectors/slack/lib/channels";
 import { getSlackClient } from "@connectors/connectors/slack/lib/slack_client";
 import { launchSlackSyncWorkflow } from "@connectors/connectors/slack/temporal/client.js";
+import { Connector, ModelId, sequelize_conn } from "@connectors/lib/models";
 import {
-  Connector,
-  ModelId,
-  sequelize_conn,
   SlackChannel,
   SlackConfiguration,
   SlackMessages,
-} from "@connectors/lib/models.js";
+} from "@connectors/lib/models/slack";
 import {
   nango_client,
   nangoDeleteConnection,

diff --git a/connectors/src/connectors/slack/lib/channels.ts b/connectors/src/connectors/slack/lib/channels.ts
@@ -1,12 +1,8 @@
 import { CodedError, ErrorCode, WebAPIPlatformError } from "@slack/web-api";
 import { Channel } from "@slack/web-api/dist/response/ConversationsListResponse";
 
-import {
-  Connector,
-  ModelId,
-  sequelize_conn,
-  SlackChannel,
-} from "@connectors/lib/models";
+import { Connector, ModelId, sequelize_conn } from "@connectors/lib/models";
+import { SlackChannel } from "@connectors/lib/models/slack";
 import { Err, Ok, Result } from "@connectors/lib/result";
 import logger from "@connectors/logger/logger";
 import { ConnectorPermission } from "@connectors/types/resources";

diff --git a/connectors/src/connectors/slack/temporal/activities.ts b/connectors/src/connectors/slack/temporal/activities.ts
@@ -29,12 +29,8 @@ import {
   upsertToDatasource,
 } from "@connectors/lib/data_sources";
 import { WorkflowError } from "@connectors/lib/error";
-import {
-  Connector,
-  ModelId,
-  SlackChannel,
-  SlackMessages,
-} from "@connectors/lib/models";
+import { Connector, ModelId } from "@connectors/lib/models";
+import { SlackChannel, SlackMessages } from "@connectors/lib/models/slack";
 import {
   reportInitialSyncProgress,
   syncSucceeded,