diff --git a/.gitignore b/.gitignore index b77e067..ff780b6 100644 --- a/.gitignore +++ b/.gitignore @@ -13,3 +13,7 @@ jackets/* *.png log/* params_secret.py + +# PyClipse project files +.project +.pydevproject diff --git a/alllog_sync.py b/alllog_sync.py new file mode 100644 index 0000000..116cbdd --- /dev/null +++ b/alllog_sync.py @@ -0,0 +1,259 @@ +import pickle +import os +import argparse +import shutil +import sys +from os import path +from PIL import Image +from datetime import datetime +from sdvxh_classes import OnePlayData +from gen_summary import GenSummary +import xml.etree.ElementTree as ET + + +def loadSongList(songList): + ret = None + with open(f'{songList}/musiclist.pkl', 'rb') as f: + ret = pickle.load(f) + return ret + +def loadPlaysList(allogFolder): + ret = None + with open(f'{allogFolder}/alllog.pkl', 'rb') as f: + ret = pickle.load(f) + return ret + + +def save(dat:dict, allogFolder): + with open(f'{allogFolder}/alllog.pkl', 'wb') as f: + pickle.dump(dat, f) + + +def isSongInLog(songLog, songToSearch): + + songFound = False + songExistOnDate = False + for songFromLog in songLog: + if songFromLog.title == songToSearch.title and songFromLog.date == songToSearch.date: +# print(f'Song {songToSearch.title} already exists on file') + songFound = True + break + elif songFromLog.title == songToSearch.title: + songLogDate = songFromLog.date.split('_')[0] + songLogTime = datetime.strptime(songFromLog.date.split('_')[1], '%H%M%S') + + if not "_" in songToSearch.date or len(songToSearch.date.split('_')) < 2 : + print(f'Mallformed song data: {songToSearch.disp()}') + return True + + songSSDate = songToSearch.date.split('_')[0] + #print(f'Searching for {songToSearch.title}') + songSSTime = datetime.strptime(songToSearch.date.split('_')[1], '%H%M%S') + + diferenceInSeconds = abs((songSSTime - songLogTime).total_seconds()) + + if songLogDate == songSSDate and diferenceInSeconds < 120: + #print(f'Song {songToSearch.title} already exists on file') + songFound = True + break + else: + #print(f'Song \'{songToSearch.title}\' already exists on file but with different date: {songFromLog.date} in log vs {songToSearch.date} in screenshot ({diferenceInSeconds}s difference)') + songExistOnDate = True + + + if not songFound and not songExistOnDate: + print(f'Song \'{songToSearch.title}\' is new!') + return False + elif not songFound and songExistOnDate : + #print(f'Song \'{songToSearch.title}\' already exists but with another date.') + return True + + return True + +# TODO: Find a way to extract the data from a result screenshot without data in the filename +def parse_unparsed_results_screen (resultsFilename): + img = Image.open(os.path.abspath(f'{rootFolder}/{playScreenshotFileName}')) + parts = genSummary.cut_result_parts(img) + genSummary.ocr() + dif = genSummary.difficulty + +def print_logo(): + print(' _ __ __ _ ____ ') + print('| | ___ __ _ | \\/ |_ _ ___(_) ___ / ___| _ _ _ __ ___ ') + print('| | / _ \\ / _` | | |\\/| | | | / __| |/ __| \\___ \\| | | | \'_ \\ / __|') + print('| |__| (_) | (_| | | | | | |_| \\__ \\ | (__ ___) | |_| | | | | (__ ') + print('|_____\\___/ \\__, | |_| |_|\\__,_|___/_|\\___| |____/ \\__, |_| |_|\\___|') + print(' |___/ |___/ ') + +def main(songLogFolder, resultsFolder): + + print_logo() + + if os.path.isdir(resultsFolder) : + rootFolder = resultsFolder + else : + print(f'Cannot run log sync: results folder \'{resultsFolder}\' is not a folder', file=sys.stderr) + exit(1) + + if os.path.isdir(songLogFolder) : + timestamp = datetime.now().strftime('%Y%m%d_%H%M%S') + backupLogFile = 'alllog.pkl.'+timestamp + print(f'Backuping log file to {backupLogFile}') + shutil.copyfile(f'{songLogFolder}/alllog.pkl', f'{songLogFolder}/{backupLogFile}') + + songLog = loadPlaysList(songLogFolder) + else : + print(f'Cannot run log sync: alllog folder \'{songLogFolder}\' is not a folder', file=sys.stderr) + exit(1) + + + print('Initialising OCR...') + # When running manually, call in the settings yourself to be able to run from the IDE + start = datetime(year=2023, month=10, day=15, hour=0) + genSummary = GenSummary(start, rootFolder + '/sync', 'true', 255, 2) + + print(f'Processing {len(os.listdir(rootFolder))} files from folder \'{rootFolder}\'') + + updatedSongs = 0 + processedFiles = 0 + for playScreenshotFileName in os.listdir(rootFolder): + # We ignore files which are a summary and are not png + if playScreenshotFileName.find('summary') > 0 : + continue + + if playScreenshotFileName.find('png') < 0 : + continue + + if not playScreenshotFileName.startswith("sdvx") : + continue + + nameSplits = playScreenshotFileName.split("_") + + songTitle = '' + for i in range(1,len(nameSplits)) : + + # Read all chunks as song title until we hit and difficulty identifier + if nameSplits[i] != 'NOV' and nameSplits[i] != 'ADV' and nameSplits[i] != 'EXH' : + songTitle += nameSplits[i] + ' ' + lastIndexOfName = i + continue + else : + break; + + # Set the rest of the data based on offset of the last chunk of the title + dif = nameSplits[lastIndexOfName+1] + + # If the chunk after the difficulty is 'class' we know it's a screenshot of the Skill Analyser mode and we skip that chunk + if nameSplits[lastIndexOfName+2] == 'class' : + lastIndexOfName+=1 + + lamp = nameSplits[lastIndexOfName+2] + + # It can happen that the score is empty and we have a file of type + # sdvx_プナイプナイたいそう_NOV_failed__20250111_173755 + # In the case, consider the score 0 otherwise things might break later + # if the playDate chunks are not assigned correctly + if nameSplits[lastIndexOfName+3] == '' : + score = 0 + else : + score = nameSplits[lastIndexOfName+3] + + playDate = nameSplits[lastIndexOfName+4]+'_'+nameSplits[lastIndexOfName+5] + playDate = playDate.removesuffix('.png') + + #print(f'Read from file: {songTitle} / {dif} / {lamp} / {score} / {playDate}') + + if songTitle != '': + + img = Image.open(os.path.abspath(f'{rootFolder}/{playScreenshotFileName}')) + scoreFromImage = genSummary.get_score(img) + + songFromScreenshot = OnePlayData(songTitle.removesuffix(' '), scoreFromImage[0], scoreFromImage[1], lamp, dif.lower(), playDate.removesuffix('.png_')) + + # If the song is not in the long, with a tolerance of 120 seconds, add it to the log + if not isSongInLog(songLog, songFromScreenshot): + songLog.append(songFromScreenshot) + updatedSongs += 1 + + processedFiles += 1 + if processedFiles % 100 == 0: + print(f'{processedFiles} files processed...') + + print(f'Update song log with {updatedSongs} songs out of {processedFiles} valid files') + save(songLog, songLogFolder) + + +def findSongRating(songFromLog, songList): + + rating = "n/a" + + # Find the numeric value of the song rating based on it's difficulty category + for songTitle in songList['titles'] : + if songTitle == songFromLog.title : + song = songList['titles'][songTitle] + if songFromLog.difficulty == 'nov' : + rating = song[3] + elif songFromLog.difficulty == 'adv' : + rating = song[4] + elif songFromLog.difficulty == 'exh' : + rating = song[5] + else : + rating = song[6] + break + + return str(rating) + +def dump(songLogFolder, songListFolder): + + songLog = loadPlaysList(songLogFolder) + songList = loadSongList(songListFolder) + + songListElement = ET.Element("songList") + xmlTree = ET.ElementTree(songListElement) + plays={} + + print(f'Dumping {len(songLog)} song plays to XML...') + for songFromLog in songLog: + + title = songFromLog.title.replace("'","\"") + + rating = findSongRating(songFromLog, songList) + songHash = str(hash(title+"_"+songFromLog.difficulty+"_"+rating)) + + existingNode = plays.get(songHash,None) + + #Format the date to more similar to ISO + songDate = datetime.strptime(songFromLog.date, '%Y%m%d_%H%M%S') + formatted_date = songDate.strftime("%Y-%m-%d %H:%M:%S") + + # If we already added this song, create new "play" entry under the same song and difficulty / rating + if existingNode is not None: + ET.SubElement(existingNode,"play",score=str(songFromLog.cur_score), lamp=songFromLog.lamp, date=formatted_date) + else : + songNode = ET.SubElement(songListElement, "song", title=title) + playsNode = ET.SubElement(songNode,"plays", difficulty=songFromLog.difficulty, rating=rating) + ET.SubElement(playsNode,"play",score=str(songFromLog.cur_score), lamp=songFromLog.lamp, date=formatted_date) + plays[songHash] = playsNode + + + print(f'Writing XML to {songLogFolder}/played_songs.xml') + ET.indent(xmlTree, space="\t", level=0) + xmlTree.write(songLogFolder+"/played_songs.xml",encoding="UTF-8",xml_declaration=True) + + +if __name__ == '__main__': + + parser = argparse.ArgumentParser(description='Reads the sdvx results folders and re-inserts missing songs into the alllog.pkl') + parser.add_argument('--songLog', required=True, help='The directory containing the alllog (alllog.pkl) file') + parser.add_argument('--results', required=True, help='The directory containing the result screenshots') + parser.add_argument('--dump', required=False, help='Dumps the alllog.pkl into an xml file', action='store_true') + parser.add_argument('--songList', required=False, help='The directory containing the song list (musiclist.pkl) file, only used with the --dump option') + + args = parser.parse_args() + main(args.songLog, args.results) + + if args.dump : + dump(args.songLog, args.songList) + + + diff --git a/gen_summary.py b/gen_summary.py index 8d4a2b2..c7fd86e 100644 --- a/gen_summary.py +++ b/gen_summary.py @@ -28,17 +28,36 @@ SWVER = "v?.?.?" class GenSummary: - def __init__(self, now): + + def __init__(self, now, autosave_dir=None, ignore_rankD=None, logpic_bg_alpha=None, log_maxnum=None): self.start = now self.result_parts = False self.difficulty = False self.load_settings() self.load_hashes() - self.savedir = self.settings['autosave_dir'] - self.ignore_rankD = self.settings['ignore_rankD'] - self.alpha = self.settings['logpic_bg_alpha'] - self.max_num = self.params['log_maxnum'] + + if autosave_dir : + self.savedir = autosave_dir + else : + self.savedir = self.settings['autosave_dir'] + + if ignore_rankD : + self.ignore_rankD = ignore_rankD + else : + self.ignore_rankD = self.settings['ignore_rankD'] + + if logpic_bg_alpha: + self.alpha = logpic_bg_alpha + else : + self.alpha = self.settings['logpic_bg_alpha'] + + if log_maxnum : + self.max_num = log_maxnum + else : + self.max_num = self.params['log_maxnum'] + print(now, self.savedir) + def load_settings(self): try: