[pull] main from dataease:main

backend/apps/chat/api/chat.py

@@ -21,8 +21,8 @@
 from common.core.deps import CurrentAssistant, SessionDep, CurrentUser, Trans
 from common.utils.command_utils import parse_quick_command
 from common.utils.data_format import DataFormat
-from sqlbot_xpack.audit.models.log_model import OperationType, OperationModules
-from sqlbot_xpack.audit.schemas.logger_decorator import LogConfig, system_log
+from common.audit.models.log_model import OperationType, OperationModules
+from common.audit.schemas.logger_decorator import LogConfig, system_log
 
 router = APIRouter(tags=["Data Q&A"], prefix="/chat")
 

backend/apps/dashboard/api/dashboard_api.py

@@ -3,8 +3,8 @@
 from apps.dashboard.crud.dashboard_service import list_resource, load_resource, \
     create_resource, create_canvas, validate_name, delete_resource, update_resource, update_canvas
 from apps.dashboard.models.dashboard_model import CreateDashboard, BaseDashboard, QueryDashboard, DashboardResponse
-from sqlbot_xpack.audit.models.log_model import OperationType, OperationModules
-from sqlbot_xpack.audit.schemas.logger_decorator import LogConfig, system_log
+from common.audit.models.log_model import OperationType, OperationModules
+from common.audit.schemas.logger_decorator import LogConfig, system_log
 from common.core.deps import SessionDep, CurrentUser
 
 router = APIRouter(tags=["dashboard"], prefix="/dashboard")

backend/apps/data_training/api/data_training.py

@@ -19,8 +19,8 @@
 from common.core.deps import SessionDep, CurrentUser, Trans
 from common.utils.data_format import DataFormat
 from common.utils.excel import get_excel_column_count
-from sqlbot_xpack.audit.models.log_model import OperationType, OperationModules
-from sqlbot_xpack.audit.schemas.logger_decorator import LogConfig, system_log
+from common.audit.models.log_model import OperationType, OperationModules
+from common.audit.schemas.logger_decorator import LogConfig, system_log
 
 router = APIRouter(tags=["SQL Examples"], prefix="/system/data-training")
 

backend/apps/datasource/api/datasource.py

@@ -28,8 +28,8 @@
 from ..crud.table import get_tables_by_ds_id
 from ..models.datasource import CoreDatasource, CreateDatasource, TableObj, CoreTable, CoreField, FieldObj, \
     TableSchemaResponse, ColumnSchemaResponse, PreviewResponse
-from sqlbot_xpack.audit.models.log_model import OperationType, OperationModules
-from sqlbot_xpack.audit.schemas.logger_decorator import LogConfig, system_log
+from common.audit.models.log_model import OperationType, OperationModules
+from common.audit.schemas.logger_decorator import LogConfig, system_log
 
 router = APIRouter(tags=["Datasource"], prefix="/datasource")
 path = settings.EXCEL_PATH

backend/apps/datasource/api/recommended_problem.py

@@ -5,8 +5,8 @@
     save_recommended_problem, get_datasource_recommended_base
 from apps.datasource.models.datasource import RecommendedProblemBase
 from common.core.deps import SessionDep, CurrentUser
-from sqlbot_xpack.audit.models.log_model import OperationType, OperationModules
-from sqlbot_xpack.audit.schemas.logger_decorator import LogConfig, system_log
+from common.audit.models.log_model import OperationType, OperationModules
+from common.audit.schemas.logger_decorator import LogConfig, system_log
 
 router = APIRouter(tags=["recommended_problem"], prefix="/recommended_problem")
 

backend/apps/db/db.py

@@ -386,7 +386,8 @@ def get_tables(ds: CoreDatasource):
                                   password=conf.password,
                                   options=f"-c statement_timeout={conf.timeout * 1000}",
                                   **extra_config_dict) as conn, conn.cursor() as cursor:
-                cursor.execute(sql.format(sql_param))
+                # Use parameterized query for security
+                cursor.execute(sql, (sql_param,))
                 res = cursor.fetchall()
                 res_list = [TableSchema(*item) for item in res]
                 return res_list
@@ -437,7 +438,8 @@ def get_fields(ds: CoreDatasource, table_name: str = None):
                                   password=conf.password,
                                   options=f"-c statement_timeout={conf.timeout * 1000}",
                                   **extra_config_dict) as conn, conn.cursor() as cursor:
-                cursor.execute(sql.format(p1, p2))
+                # Use parameterized query for security
+                cursor.execute(sql, (p1, p2))
                 res = cursor.fetchall()
                 res_list = [ColumnSchema(*item) for item in res]
                 return res_list

backend/apps/db/es_engine.py

@@ -110,7 +110,18 @@ def get_es_data_by_http(conf: DatasourceConf, sql: str):
 
     host = f'{url}/_sql?format=json'
 
-    response = requests.post(host, data=json.dumps({"query": sql}), headers=get_es_auth(conf), verify=False)
+    # Security improvement: Enable SSL certificate verification
+    # Note: In production, always set verify=True or provide path to CA bundle
+    # If using self-signed certificates, provide the cert path: verify='/path/to/cert.pem'
+    verify_ssl = True if not url.startswith('https://localhost') else False
+
+    response = requests.post(
+        host, 
+        data=json.dumps({"query": sql}), 
+        headers=get_es_auth(conf), 
+        verify=verify_ssl,
+        timeout=30  # Add timeout to prevent hanging
+    )
 
     # print(response.json())
     res = response.json()

backend/apps/system/api/aimodel.py

@@ -16,8 +16,8 @@
 from common.utils.utils import SQLBotLogUtil, prepare_model_arg
 
 router = APIRouter(tags=["system_model"], prefix="/system/aimodel")
-from sqlbot_xpack.audit.models.log_model import OperationType, OperationModules
-from sqlbot_xpack.audit.schemas.logger_decorator import LogConfig, system_log
+from common.audit.models.log_model import OperationType, OperationModules
+from common.audit.schemas.logger_decorator import LogConfig, system_log
 
 @router.post("/status", include_in_schema=False)
 @require_permissions(permission=SqlbotPermission(role=['admin'])) 

backend/apps/system/api/apikey.py

@@ -9,8 +9,8 @@
 import secrets
 
 router = APIRouter(tags=["system_apikey"], prefix="/system/apikey", include_in_schema=False)
-from sqlbot_xpack.audit.models.log_model import OperationType, OperationModules
-from sqlbot_xpack.audit.schemas.logger_decorator import LogConfig, system_log
+from common.audit.models.log_model import OperationType, OperationModules
+from common.audit.schemas.logger_decorator import LogConfig, system_log
 
 @router.get("")
 async def grid(session: SessionDep, current_user: CurrentUser) -> list[ApikeyGridItem]:

backend/apps/system/api/assistant.py

@@ -21,8 +21,8 @@
 from common.utils.utils import get_origin_from_referer, origin_match_domain
 
 router = APIRouter(tags=["system_assistant"], prefix="/system/assistant")
-from sqlbot_xpack.audit.models.log_model import OperationType, OperationModules
-from sqlbot_xpack.audit.schemas.logger_decorator import LogConfig, system_log
+from common.audit.models.log_model import OperationType, OperationModules
+from common.audit.schemas.logger_decorator import LogConfig, system_log
 
 
 @router.get("/info/{id}", include_in_schema=False)

backend/apps/system/api/login.py

@@ -12,8 +12,8 @@
 from common.core.schemas import Token
 from sqlbot_xpack.authentication.manage import logout as xpack_logout
 
-from sqlbot_xpack.audit.models.log_model import OperationType, OperationModules
-from sqlbot_xpack.audit.schemas.logger_decorator import system_log, LogConfig
+from common.audit.models.log_model import OperationType, OperationModules
+from common.audit.schemas.logger_decorator import system_log, LogConfig
 
 router = APIRouter(tags=["login"], prefix="/login")
 

backend/apps/system/api/parameter.py

@@ -6,8 +6,8 @@
 from common.core.deps import SessionDep
 
 router = APIRouter(tags=["system/parameter"], prefix="/system/parameter", include_in_schema=False)
-from sqlbot_xpack.audit.models.log_model import OperationType, OperationModules
-from sqlbot_xpack.audit.schemas.logger_decorator import LogConfig, system_log
+from common.audit.models.log_model import OperationType, OperationModules
+from common.audit.schemas.logger_decorator import LogConfig, system_log
 
 @router.get("/login")
 async def get_login_args(session: SessionDep) -> list[SysArgModel]:

backend/apps/system/api/user.py

@@ -1,16 +1,16 @@
 from collections import defaultdict
 from typing import Optional
-from fastapi import APIRouter, Path, Query
-from pydantic import Field
+from fastapi import APIRouter, File, Path, Query, UploadFile
 from sqlmodel import SQLModel, or_, select, delete as sqlmodel_delete
 from apps.system.crud.user import check_account_exists, check_email_exists, check_email_format, check_pwd_format, get_db_user, single_delete, user_ws_options
+from apps.system.crud.user_excel import batchUpload, downTemplate, download_error_file
 from apps.system.models.system_model import UserWsModel, WorkspaceModel
 from apps.system.models.user import UserModel
 from apps.system.schemas.auth import CacheName, CacheNamespace
 from apps.system.schemas.permission import SqlbotPermission, require_permissions
 from apps.system.schemas.system_schema import PwdEditor, UserCreator, UserEditor, UserGrid, UserInfoDTO, UserLanguage, UserStatus, UserWs
-from sqlbot_xpack.audit.models.log_model import OperationType, OperationModules
-from sqlbot_xpack.audit.schemas.logger_decorator import LogConfig, system_log
+from common.audit.models.log_model import OperationType, OperationModules
+from common.audit.schemas.logger_decorator import LogConfig, system_log
 from common.core.deps import CurrentUser, SessionDep, Trans
 from common.core.pagination import Paginator
 from common.core.schemas import PaginatedResponse, PaginationParams
@@ -21,6 +21,23 @@
 
 router = APIRouter(tags=["system_user"], prefix="/user")
 
+
+@router.get("/template", include_in_schema=False)
+@require_permissions(permission=SqlbotPermission(role=['admin']))
+async def templateExcel(trans: Trans):
+    return await downTemplate(trans)
+
+@router.post("/batchImport", include_in_schema=False)
+@require_permissions(permission=SqlbotPermission(role=['admin']))
+async def upload_excel(session: SessionDep, trans: Trans, current_user: CurrentUser, file: UploadFile = File(...)):
+    return await batchUpload(session, trans, file)
+
+
+@router.get("/errorRecord/{file_id}", include_in_schema=False)
+@require_permissions(permission=SqlbotPermission(role=['admin']))
+async def download_error(file_id: str):
+    return download_error_file(file_id)
+
 @router.get("/info", summary=f"{PLACEHOLDER_PREFIX}system_user_current_user", description=f"{PLACEHOLDER_PREFIX}system_user_current_user_desc")
 async def user_info(current_user: CurrentUser) -> UserInfoDTO:
     return current_user
@@ -301,4 +318,4 @@ async def statusChange(session: SessionDep, current_user: CurrentUser, trans: Tr
         return {"message": "status not supported"}
     db_user: UserModel = get_db_user(session=session, user_id=statusDto.id)
     db_user.status = status
-    session.add(db_user)
+    session.add(db_user)

backend/apps/system/api/workspace.py

@@ -8,8 +8,8 @@
 from apps.system.models.user import UserModel
 from apps.system.schemas.permission import SqlbotPermission, require_permissions
 from apps.system.schemas.system_schema import UserWsBase, UserWsDTO, UserWsEditor, UserWsOption, WorkspaceUser
-from sqlbot_xpack.audit.models.log_model import OperationType, OperationModules
-from sqlbot_xpack.audit.schemas.logger_decorator import system_log, LogConfig
+from common.audit.models.log_model import OperationType, OperationModules
+from common.audit.schemas.logger_decorator import system_log, LogConfig
 from common.core.deps import CurrentUser, SessionDep, Trans
 from common.core.pagination import Paginator
 from common.core.schemas import PaginatedResponse, PaginationParams