flag to read chat export; exclude emojis, double spaces etc; pretty print trigrams

.gitignore

@@ -1,3 +1,4 @@
 *.json
 *.blob
 *.txt
+.idea

cybercum.go

@@ -1,22 +1,13 @@
 package cybercum
 
 import (
-	"github.com/dyvdev/cybercum/tgbot"
 	"log"
 	"os"
 	"os/signal"
 	"syscall"
-)
 
-func ReadBot(cfgFile string) {
-	log.Println("starting...")
-	bot := tgbot.NewBot()
-	log.Println("reading...")
-	//bot.Swatter.ReadFile("mh.txt")
-	log.Println("saving...")
-	bot.SaveDump()
-	log.Println("done...")
-}
+	"github.com/dyvdev/cybercum/tgbot"
+)
 
 func CleanBot() {
 	bot := tgbot.NewBot()

go.mod

@@ -9,4 +9,4 @@ require (
 
 require github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0 // indirect
 
-replace github.com/go-telegram-bot-api/telegram-bot-api/v5 => /home/askival/go/src/github.com/telegram-bot-api
+//replace github.com/go-telegram-bot-api/telegram-bot-api/v5 => /home/user/go/src/github.com/telegram-bot-api

go.sum

@@ -1,3 +1,5 @@
+github.com/go-telegram-bot-api/telegram-bot-api/v5 v5.5.1 h1:wG8n/XJQ07TmjbITcGiUaOtXxdrINDz1b0J1w0SzqDc=
+github.com/go-telegram-bot-api/telegram-bot-api/v5 v5.5.1/go.mod h1:A2S0CWkNylc2phvKXWBBdD3K0iGnDBGbzRpISP2zBl8=
 github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0 h1:OdAsTTz6OkFY5QxjkYwrChwuRruF69c169dPK26NUlk=
 github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
 modernc.org/mathutil v1.5.0 h1:rV0Ko/6SfM+8G+yKiyI830l3Wuz1zRutdslNoQ0kfiQ=

main/main.go

@@ -1,39 +1,57 @@
 package main
 
 import (
-	cum "github.com/dyvdev/cybercum"
-	"github.com/dyvdev/cybercum/swatter"
-	"github.com/dyvdev/cybercum/utils"
+	"flag"
 	"log"
 	"math/rand"
 	"os"
+	"path/filepath"
+	"strings"
 	"time"
+
+	cum "github.com/dyvdev/cybercum"
+	"github.com/dyvdev/cybercum/swatter"
+	"github.com/dyvdev/cybercum/utils"
 )
 
 func main() {
+	var histPath *string
+	histPath = flag.String("rhist", "", "read chat dump, overwrite blob and exit")
+	flag.Parse()
+	if len(*histPath) != 0 {
+		ChatHistoryGen(*histPath)
+		return
+	}
 	rand.Seed(time.Now().UnixNano())
-	//cum.ReadBot()
 	cum.RunBot()
-	//cum.CleanBot()
-	//test()
-	//testChatHistoryGen()
 }
 
-func testChatHistoryGen() {
+func ChatHistoryGen(historyFile string) {
 	sw := &swatter.DataStorage{}
-	data := utils.GetTgData("tghistory.json")
-	file, err := os.Create("tghistory.txt")
+	data := utils.GetTgData(historyFile)
+	baseName := filepath.Base(historyFile)
+	cleanName := strings.Split(baseName, ".")
+	if len(cleanName) < 2 {
+		return
+	}
+
+	textDumpName := cleanName[0] + ".txt"
+	file, err := os.Create(textDumpName)
 	if err != nil {
 		log.Fatal(err)
 		return
 	}
 	defer file.Close()
 	for _, str := range data {
-		sw.ParseText(str)
-		file.WriteString(str)
+		newStr := sw.ParseText(str)
+		if newStr == "" {
+			continue
+		}
+
+		file.WriteString(newStr + " ")
 	}
-	log.Print(sw.GenerateText("", 15))
 }
+
 func test() {
 	sw := &swatter.DataStorage{}
 	sw.ReadFile("mh.txt")

swatter/swatter.go

@@ -3,13 +3,14 @@ package swatter
 import (
 	"bufio"
 	"encoding/gob"
-	"github.com/dyvdev/cybercum/utils"
 	"log"
 	"math/rand"
 	"os"
 	"regexp"
 	"strconv"
 	"strings"
+
+	"github.com/dyvdev/cybercum/utils"
 )
 
 type Trigram [3]string
@@ -139,11 +140,36 @@ func (data DataStorage) GenerateText(msg string, length int) string {
 			last2Words[1] = trigram[2]
 		}
 	}
+	for i := range text {
+		text[i] = cleanWord(text[i])
+
+	}
 	return strings.Join(text, " ")
 }
 
-func (data DataStorage) ParseText(text string) {
-	text = strings.ToLower(regexp.MustCompile(`\.|,|;|!|\?`).ReplaceAllString(text, ""))
+func cleanWord(word string) string {
+	if len(word) < 2 {
+		return ""
+	}
+	word = utils.TrimWord(word)
+	if word[0] == '"' && word[len(word)-1] != '"' {
+		word = strings.ReplaceAll(word, "\"", "")
+	}
+	if word[0] == '\'' && word[len(word)-1] != '\'' {
+		word = strings.ReplaceAll(word, "'", "")
+	}
+	word = strings.Trim(word, "\"\\n\\")
+	return word
+}
+
+func (data DataStorage) ParseText(text string) string {
+	text = strings.ToLower(regexp.MustCompile(`\.|,|;|!|\?|\t`).ReplaceAllString(text, ""))
+	text = strings.ToLower(regexp.MustCompile(`[^a-zA-Zа-яА-Я\s\d]`).ReplaceAllString(text, ""))
+	text = strings.ToLower(regexp.MustCompile(`\n{2,}`).ReplaceAllString(text, "\n"))
+	text = strings.ToLower(regexp.MustCompile(" {2,}").ReplaceAllString(text, " "))
+	text = strings.ToLower(regexp.MustCompile(" {2,}").ReplaceAllString(text, " "))
+	text = strings.TrimSpace(text)
+
 	words := strings.Split(text, " ")
 	var trimmedWords []string
 	last := ""
@@ -159,6 +185,7 @@ func (data DataStorage) ParseText(text string) {
 			data.AddTrigram(Trigram{trimmedWords[i], trimmedWords[i+1], trimmedWords[i+2]})
 		}
 	}
+	return text
 }
 
 func (data DataStorage) ReadFile(filename string) error {
@@ -199,13 +226,11 @@ func (data DataStorage) SaveDump(filename string) {
 func (data DataStorage) LoadDump(filename string) error {
 	f, err := os.Open(filename)
 	if err != nil {
-		log.Println(err)
 		return err
 	}
 	defer f.Close()
 	dec := gob.NewDecoder(f)
 	if err := dec.Decode(&data); err != nil {
-		log.Fatal(err)
 		return err
 	}
 	return nil

tgbot/tgbot.go

@@ -2,15 +2,16 @@ package tgbot
 
 import (
 	"encoding/json"
-	"github.com/dyvdev/cybercum/swatter"
-	tgbotapi "github.com/go-telegram-bot-api/telegram-bot-api/v5"
 	"log"
 	"math/rand"
-	"modernc.org/mathutil"
 	"os"
 	"strconv"
 	"strings"
 	"time"
+
+	"github.com/dyvdev/cybercum/swatter"
+	tgbotapi "github.com/go-telegram-bot-api/telegram-bot-api/v5"
+	"modernc.org/mathutil"
 )
 
 const (
@@ -186,7 +187,8 @@ func (bot *Bot) IsCum(message *tgbotapi.Message) bool {
 		ChatConfigWithUser: tgbotapi.ChatConfigWithUser{
 			ChatID:             message.Chat.ID,
 			SuperGroupUsername: "",
-			UserID:             message.From.ID},
+			UserID:             message.From.ID,
+		},
 	})
 	if err == nil {
 		chat := bot.Chats[message.Chat.ID]
@@ -218,14 +220,14 @@ func (bot *Bot) ShowUpdateInfo(update tgbotapi.Update) {
 
 func (bot *Bot) GenerateMessage(message *tgbotapi.Message) tgbotapi.Chattable {
 	msg := bot.Swatter[message.Chat.ID].GenerateText(message.Text, bot.Chats[message.Chat.ID].SemenLength)
-	threadId := 0
-	if message.Chat.IsForum && message.MessageThreadID != 0 {
-		threadId = message.MessageThreadID
-	}
+	//threadId := 0
+	//if message.Chat.IsForum && message.MessageThreadID != 0 {
+	//	threadId = message.MessageThreadID
+	//}
 	return tgbotapi.MessageConfig{
 		BaseChat: tgbotapi.BaseChat{
-			ChatID:           message.Chat.ID,
-			MessageThreadID:  threadId,
+			ChatID: message.Chat.ID,
+			//MessageThreadID:  threadId,
 			ReplyToMessageID: 0,
 		},
 		Text:                  msg,
@@ -268,14 +270,14 @@ func (bot *Bot) ReplyNefren(message *tgbotapi.Message) {
 func (bot *Bot) SendFixedPhrase(message *tgbotapi.Message) {
 	chat := bot.Chats[message.Chat.ID]
 	if len(chat.FixedPhrases) != 0 {
-		threadId := 0
-		if message.Chat.IsForum && message.MessageThreadID != 0 {
-			threadId = message.MessageThreadID
-		}
+		//threadId := 0
+		//if message.Chat.IsForum && message.MessageThreadID != 0 {
+		//	threadId = message.MessageThreadID
+		//}
 		bot.SendMessage(tgbotapi.MessageConfig{
 			BaseChat: tgbotapi.BaseChat{
-				ChatID:           message.Chat.ID,
-				MessageThreadID:  threadId,
+				ChatID: message.Chat.ID,
+				//MessageThreadID:  threadId,
 				ReplyToMessageID: 0,
 			},
 			Text:                  chat.FixedPhrases[rand.Intn(len(chat.FixedPhrases)-1)],
@@ -352,6 +354,19 @@ func (bot *Bot) SaveDump() {
 	}
 }
 
+func (bot *Bot) SaveDumpFromTextFile(filepath string) {
+	cfgJson, _ := json.Marshal(bot.Chats)
+	err := os.WriteFile("chats.json", cfgJson, 0644)
+	if err != nil {
+		log.Fatal("Error during saving chats: ", err)
+	}
+
+	for key, chat := range bot.Chats {
+		bot.Swatter[key].ReadFile(filepath)
+		bot.Swatter[key].SaveDump(chat.ChatName + ".blob")
+	}
+}
+
 func (bot *Bot) LoadDump() {
 	log.Println("reading chats...")
 	content, err := os.ReadFile("chats.json")
@@ -383,7 +398,6 @@ func (bot *Bot) LoadDump() {
 	if needToSave {
 		bot.SaveDump()
 	}
-	//bot.FixChats()
 	log.Println("reading chats...done")
 }
 
@@ -459,14 +473,12 @@ func (bot *Bot) Dumper(done <-chan bool) {
 	ticker := time.NewTicker(dumpTick)
 	go func() {
 		for {
-			//time.Sleep(25 * time.Millisecond)
 			select {
 			case <-done:
 				bot.BotApi.StopReceivingUpdates()
 				return
 			case <-ticker.C:
 				bot.SaveDump()
-			default:
 			}
 		}
 	}()