Skip to content

Commit

Permalink
feat: 텍스트 전처리 로직 수정
Browse files Browse the repository at this point in the history
- 입력 타입에 맞춰서 반환
  • Loading branch information
myeolinmalchi committed Jan 29, 2025
1 parent 5002aff commit dfb31ad
Showing 1 changed file with 1 addition and 4 deletions.
5 changes: 1 addition & 4 deletions utils/text.py
Original file line number Diff line number Diff line change
@@ -4,8 +4,6 @@
from kss import Kss
from kss._utils.logger import logger

logger.addFilter(lambda record: False)

split_sentences = Kss("split_sentences")
normalize = Kss("normalize")

@@ -70,9 +68,8 @@ def preprocess(text: str | List[str], **kwargs) -> str | List[str]:
texts.append(temp)
"""텍스트 전처리"""

#tasks = [run_in_threadpool(preprocess_single, t) for t in text]
cleaned = [preprocess_single(t) for t in texts]
return cleaned
return cleaned[0] if isinstance(text, str) else cleaned


def split_chunks(query: str, max_chunk_length: int = 1000, offset: int = 100):

0 comments on commit dfb31ad

Please sign in to comment.