Merge remote-tracking branch 'origin/develop'

Signed-off-by: Denis Bykhov <bykhov.denis@gmail.com>

dev/doc-import-tool/src/index.ts

@@ -54,17 +54,11 @@ export function docImportTool (): void {
 
   const uploadUrl = process.env.UPLOAD_URL ?? '/files'
 
-  const mongodbUri = process.env.MONGO_URL
-  if (mongodbUri === undefined) {
-    console.log('Please provide mongodb url')
-    process.exit(1)
-  }
-
   setMetadata(serverClientPlugin.metadata.Endpoint, accountUrl)
   setMetadata(serverToken.metadata.Secret, serverSecret)
 
-  async function withStorage (mongodbUri: string, f: (storageAdapter: StorageAdapter) => Promise<any>): Promise<void> {
-    const adapter = buildStorageFromConfig(storageConfigFromEnv(), mongodbUri)
+  async function withStorage (f: (storageAdapter: StorageAdapter) => Promise<any>): Promise<void> {
+    const adapter = buildStorageFromConfig(storageConfigFromEnv())
     try {
       await f(adapter)
     } catch (err: any) {
@@ -94,7 +88,7 @@ export function docImportTool (): void {
           }, space: ${cmd.space}, backend: ${cmd.backend}`
         )
 
-        await withStorage(mongodbUri, async (storageAdapter) => {
+        await withStorage(async (storageAdapter) => {
           const workspaceId = getWorkspaceId(workspace)
 
           const config: Config = {

dev/docker-compose.yaml

@@ -106,7 +106,6 @@ services:
     environment:
       # - WS_OPERATION=create
       - SERVER_SECRET=secret
-      - MONGO_URL=${MONGO_URL}
       - DB_URL=${MONGO_URL}
       # - DB_URL=postgresql://postgres:example@postgres:5432
       - SES_URL=
@@ -132,7 +131,6 @@ services:
     environment:
       # - WS_OPERATION=create
       - SERVER_SECRET=secret
-      - MONGO_URL=${MONGO_URL}
       - DB_URL=postgresql://postgres:example@postgres:5432
       - SES_URL=
       - REGION=pg
@@ -160,8 +158,6 @@ services:
       - COLLABORATOR_PORT=3078
       - SECRET=secret
       - ACCOUNTS_URL=http://host.docker.internal:3000
-      - MONGO_URL=${MONGO_URL}
-      - 'MONGO_OPTIONS={"appName":"collaborator","maxPoolSize":2}'
       - STORAGE_CONFIG=${STORAGE_CONFIG}
     restart: unless-stopped
   front:
@@ -178,11 +174,8 @@ services:
       - 8087:8080
       - 8088:8080
     environment:
-      - UV_THREADPOOL_SIZE=10
       - SERVER_PORT=8080
       - SERVER_SECRET=secret
-      - MONGO_URL=${MONGO_URL}
-      - 'MONGO_OPTIONS={"appName":"front","maxPoolSize":1}'
       - ACCOUNTS_URL=http://host.docker.internal:3000
       - UPLOAD_URL=/files
       - ELASTIC_URL=http://host.docker.internal:9200
@@ -297,8 +290,6 @@ services:
       - 4005:4005
     environment:
       - SECRET=secret
-      - MONGO_URL=${MONGO_URL}
-      - 'MONGO_OPTIONS={"appName":"print","maxPoolSize":1}'
       - STORAGE_CONFIG=${STORAGE_CONFIG}
     deploy:
       resources:
@@ -316,8 +307,6 @@ services:
       - ../services/sign/pod-sign/debug/branding.json:/var/cfg/branding.json
     environment:
       - SECRET=secret
-      - MONGO_URL=${MONGO_URL}
-      - 'MONGO_OPTIONS={"appName":"sign","maxPoolSize":1}'
       - MINIO_ENDPOINT=minio
       - MINIO_ACCESS_KEY=minioadmin
       - ACCOUNTS_URL=http://host.docker.internal:3000

dev/prod/src/analytics/posthog.ts

@@ -4,7 +4,12 @@ import posthog from 'posthog-js'
 export class PosthogAnalyticProvider implements AnalyticProvider {
   init(config: Record<string, any>): boolean {
     if (config.POSTHOG_API_KEY !== undefined && config.POSTHOG_API_KEY !== '' && config.POSTHOG_HOST !== null) {
-      posthog.init(config.POSTHOG_API_KEY, { api_host: config.POSTHOG_HOST })
+      posthog.init(config.POSTHOG_API_KEY, {
+        api_host: config.POSTHOG_HOST,
+        autocapture: false,
+        capture_pageview: false,
+        capture_pageleave: false
+      })
       return true
     }
     return false

dev/tool/src/__start.ts

@@ -72,7 +72,6 @@ addLocation(serverDriveId, () => import('@hcengineering/server-drive-resources')
 addLocation(serverAiBotId, () => import('@hcengineering/server-ai-bot-resources'))
 
 function prepareTools (): {
-  mongodbUri: string | undefined
   dbUrl: string
   txes: Tx[]
   version: Data<Version>
@@ -84,4 +83,13 @@ function prepareTools (): {
   return { ...prepareToolsRaw(builder(enabled, disabled).getTxes()), version: getModelVersion(), migrateOperations }
 }
 
+export function getMongoDBUrl (): string {
+  const url = process.env.MONGO_URL
+  if (url === undefined) {
+    console.error('please provide mongo DB URL')
+    process.exit(1)
+  }
+  return url
+}
+
 devTool(prepareTools)

dev/tool/src/clean.ts

@@ -1270,7 +1270,7 @@ async function updateYDoc (
   doc: RelatedDocument
 ): Promise<void> {
   try {
-    const ydoc = await loadCollaborativeDoc(storage, workspaceId, _id, ctx)
+    const ydoc = await loadCollaborativeDoc(ctx, storage, workspaceId, _id)
     if (ydoc === undefined) {
       ctx.error('document content not found', { document: contentDoc._id })
       return
@@ -1284,7 +1284,7 @@ async function updateYDoc (
     })
 
     if (updatedYDoc !== undefined) {
-      await saveCollaborativeDoc(storage, workspaceId, _id, updatedYDoc, ctx)
+      await saveCollaborativeDoc(ctx, storage, workspaceId, _id, updatedYDoc)
     }
   } catch {
     // do nothing, the collaborative doc does not sem to exist yet

dev/tool/src/db.ts

@@ -18,7 +18,14 @@ import {
   type MeasureMetricsContext
 } from '@hcengineering/core'
 import { getMongoClient, getWorkspaceMongoDB } from '@hcengineering/mongo'
-import { convertDoc, createTable, getDBClient, retryTxn, translateDomain } from '@hcengineering/postgres'
+import {
+  convertDoc,
+  createTable,
+  getDBClient,
+  getDocFieldsByDomains,
+  retryTxn,
+  translateDomain
+} from '@hcengineering/postgres'
 import { getTransactorEndpoint } from '@hcengineering/server-client'
 import { generateToken } from '@hcengineering/server-token'
 import { connect } from '@hcengineering/server-tool'
@@ -54,10 +61,6 @@ export async function moveFromMongoToPG (
   client.close()
 }
 
-function escapeBackticks (str: string): string {
-  return str.replaceAll("'", "''")
-}
-
 async function moveWorkspace (
   accountDb: AccountDB,
   mongo: MongoClient,
@@ -85,6 +88,13 @@ async function moveWorkspace (
       const currentIds = new Set(current.rows.map((r) => r._id))
       console.log('move domain', domain)
       const docs: Doc[] = []
+      const fields = getDocFieldsByDomains(domain)
+      const filedsWithData = [...fields, 'data']
+      const insertFields: string[] = []
+      for (const field of filedsWithData) {
+        insertFields.push(`"${field}"`)
+      }
+      const insertStr = insertFields.join(', ')
       while (true) {
         while (docs.length < 50000) {
           const doc = (await cursor.next()) as Doc | null
@@ -95,18 +105,29 @@ async function moveWorkspace (
         if (docs.length === 0) break
         while (docs.length > 0) {
           const part = docs.splice(0, 500)
-          const vals = part
-            .map((doc) => {
-              const d = convertDoc(doc, ws.workspace)
-              return `('${d._id}', '${d.workspaceId}', '${d._class}', '${d.createdBy ?? d.modifiedBy}', '${d.modifiedBy}', ${d.modifiedOn}, ${d.createdOn ?? d.modifiedOn}, '${d.space}', ${
-                d.attachedTo != null ? `'${d.attachedTo}'` : 'NULL'
-              }, '${escapeBackticks(JSON.stringify(d.data))}')`
-            })
-            .join(', ')
+          const values: any[] = []
+          const vars: string[] = []
+          let index = 1
+          for (let i = 0; i < part.length; i++) {
+            const doc = part[i]
+            const variables: string[] = []
+            const d = convertDoc(domain, doc, ws.workspace)
+            values.push(d.workspaceId)
+            variables.push(`$${index++}`)
+            for (const field of fields) {
+              values.push(d[field])
+              variables.push(`$${index++}`)
+            }
+            values.push(d.data)
+            variables.push(`$${index++}`)
+            vars.push(`(${variables.join(', ')})`)
+          }
+          const vals = vars.join(',')
           try {
             await retryTxn(pgClient, async (client) => {
               await client.query(
-                `INSERT INTO ${translateDomain(domain)} (_id, "workspaceId", _class, "createdBy", "modifiedBy", "modifiedOn", "createdOn", space, "attachedTo", data) VALUES ${vals}`
+                `INSERT INTO ${translateDomain(domain)} ("workspaceId", ${insertStr}) VALUES ${vals}`,
+                values
               )
             })
           } catch (err) {