From 1fda5c76386d8775ce76dda6cb30664dd249608a Mon Sep 17 00:00:00 2001 From: gdiaz384 <15351882+gdiaz384@users.noreply.github.com> Date: Mon, 18 Mar 2024 15:11:42 -0700 Subject: [PATCH] The grind resumes. --- py3TranslateLLM.ini | 6 +- py3TranslateLLM.py | 185 ++++++++++++++++++++++---------- resources/chocolate.py | 171 ++++++++++++++++++----------- resources/translationEngines.py | 78 +++++++++++--- 4 files changed, 305 insertions(+), 135 deletions(-) diff --git a/py3TranslateLLM.ini b/py3TranslateLLM.ini index e236152..036b0e1 100644 --- a/py3TranslateLLM.ini +++ b/py3TranslateLLM.ini @@ -21,8 +21,8 @@ fileToTranslateEncoding=None outputFile=None outputFileEncoding=None -parsingSettingsFile=None -parsingSettingsFileEncoding=None +#parsingSettingsFile=None +#parsingSettingsFileEncoding=None promptFile=None promptFileEncoding=None @@ -62,7 +62,7 @@ readOnlyCache=False # The number of previous translations that should be sent to the translation engine to provide context for the current translation. Sane values are 2-10. Set to 0 to disable. Not all translation engines support context. Default = 4. contextHistoryLength=None # True, False. If True, store and translate lines one at a time. Disables grouping lines by delimitor and paragraph style translations. -lineByLineMode=False +#lineByLineMode=False # True, False. If True, attempt to resume previously interupted operation. No gurantees. Only checks backups made today and yesterday. resume=False diff --git a/py3TranslateLLM.py b/py3TranslateLLM.py index 2f00857..05100d3 100644 --- a/py3TranslateLLM.py +++ b/py3TranslateLLM.py @@ -34,7 +34,7 @@ # Valid options are 'Portuguese (European)' or 'Portuguese (Brazilian)' defaultPortugueseLanguage='Portuguese (European)' -defaultCacheFile='backups/cache.xlsx' + defaultAddress='http://localhost' defaultKoboldCppPort=5001 defaultPy3TranslationServerPort=14366 @@ -53,12 +53,15 @@ defaultMetadataDelimiter='_' defaultScriptSettingsFileExtension='.ini' -# Currently, this is relative to py3TranslateLLM.py, but it might also make sense to move this either relative to the target or to a system temp folder. -# There is no gurantee that being relative to the target is a sane thing to do since that depends upon runtime usage, and centralized backups also make sense. Leaving it as-is makes sense too. +# Currently, these are relative to py3TranslateLLM.py, but it might also make sense to move them either relative to the target or to a system folder intended for holding program data. +# There is no gurantee that being relative to the target is a sane thing to do since that depends upon runtime usage, and centralized backups also make sense. Leaving it as-is makes sense too as long as py3TranslateLLM is not being used as a library. If it is not being used as a library, then a centralized location under $HOME or %localappdata% makes more sense than relative to py3TranslateLLM.py. Same with the default location for the cache file. +# Maybe a good way to check for this is the name = __main__ check? +defaultExportExtension='.xlsx' +defaultCacheFile='backups/cache' + defaultExportExtension defaultBackupsFolder='backups' -translationEngines='parseOnly, koboldcpp, deepl_api_free, deepl_api_pro, deepl_web, py3translationserver, sugoi' -usageHelp=' Usage: python py3TranslateLLM --help Example: py3TranslateLLM -mode KoboldCpp -f myInputFile.ks \n Translation Engines: '+translationEngines+'.' +translationEnginesAvailable='parseOnly, koboldcpp, deepl_api_free, deepl_api_pro, deepl_web, py3translationserver, sugoi' +usageHelp=' Usage: python py3TranslateLLM --help Example: py3TranslateLLM -mode KoboldCpp -f myInputFile.ks \n Translation Engines: '+translationEnginesAvailable+'.' #import various libraries that py3TranslateLLM depends on @@ -74,15 +77,15 @@ #Technically, these two are optional for parseOnly. To support or not support such a thing... probably yes. #from collections import deque # Used to hold rolling history of translated items to use as context for new translations. -import collections # Newer syntax. For deque. -import requests # Do basic http stuff, like submitting post/get requests to APIs. Must be installed using: 'pip install requests' +import collections # Newer syntax. For deque. Used to hold rolling history of translated items to use as context for new translations. +#import requests # Do basic http stuff, like submitting post/get requests to APIs. Must be installed using: 'pip install requests' #import openpyxl # Used as the core internal data structure and to read/write xlsx files. Must be installed using pip. import resources.chocolate as chocolate # Implements openpyxl. A helper/wrapper library to aid in using openpyxl as a datastructure. import resources.dealWithEncoding as dealWithEncoding # dealWithEncoding implements the 'chardet' library which is installed with 'pip install chardet' import resources.py3TranslateLLMfunctions as py3TranslateLLMfunctions # Moved most generic functions here to increase code readability and enforce function best practices. #from resources.py3TranslateLLMfunctions import * # Do not use this syntax if at all possible. The * is fine, but the 'from' breaks everything because it copies everything instead of pointing to the original resources which makes updating library variables borderline impossible. - +import resources.translationEngines as translationEngines #Using the 'namereplace' error handler for text encoding requires Python 3.5+, so use an older one if necessary. sysVersion=int(sys.version_info[1]) @@ -104,14 +107,16 @@ # Add command line options. commandLineParser=argparse.ArgumentParser(description='Description: CLI wrapper script for various NMT and LLM models.' + usageHelp) -commandLineParser.add_argument('-mode', '--translationEngine', help='Specify translation engine to use, options=' + translationEngines+'.', type=str) +commandLineParser.add_argument('-mode', '--translationEngine', help='Specify translation engine to use, options=' + translationEnginesAvailable+'.', type=str) commandLineParser.add_argument('-f', '--fileToTranslate', help='Either the raw file to translate or the spreadsheet file to resume translating from, including path.', default=None, type=str) commandLineParser.add_argument('-fe', '--fileToTranslateEncoding', help='The encoding of the input file. Default='+str(defaultTextEncoding), default=None, type=str) commandLineParser.add_argument('-o', '--outputFile', help='The file to insert translations into, including path. Default is same as input file.', default=None, type=str) commandLineParser.add_argument('-ofe', '--outputFileEncoding', help='The encoding of the output file. Default is same as input file.', default=None, type=str) + #commandLineParser.add_argument('-pfile', '--parsingSettingsFile', help='This file defines how to parse raw text and .ks files. It is required for text and .ks files. If not specified, a template will be created.', default=None, type=str) #commandLineParser.add_argument('-pfe', '--parsingSettingsFileEncoding', help='Specify encoding for parsing definitions file, default='+str(defaultTextEncoding), default=None, type=str) + commandLineParser.add_argument('-p', '--promptFile', help='This file has the prompt for the LLM.', default=None, type=str) commandLineParser.add_argument('-pe', '--promptFileEncoding', help='Specify encoding for prompt file, default='+str(defaultTextEncoding), default=None, type=str) @@ -132,7 +137,7 @@ commandLineParser.add_argument('-c', '--cacheFile', help='The location of the cache file. Must be in .xlsx format. Default=' + str(defaultCacheFile), default=None, type=str) commandLineParser.add_argument('-nc', '--noCache', help='Disables using or updating the cache file. Default=Use the cache file to fill in previously translated entries and update it with new entries.', action='store_true') commandLineParser.add_argument('-cam', '--cacheAnyMatch', help='Use all translation engines when considering the cache. Default=Only consider the current translation engine as valid for cache hits.', action='store_true') -commandLineParser.add_argument('-oc', '--overrideWithCache', help='Do not retranslate lines, but override any already translated lines in the spreadsheet with results in the cache. Default=Do not override already translated lines.', action='store_true') +commandLineParser.add_argument('-oc', '--overrideWithCache', help='Do not retranslate lines, but override any already translated lines in the spreadsheet with results taken from the cache. Default=Do not override already translated lines.', action='store_true') commandLineParser.add_argument('-rt', '--reTranslate', help='Translate all lines even if they already have translations or are in the cache. Update the cache with the new translations. Default=Do not retranslate and use the cache to fill in previously translated lines.', action='store_true') commandLineParser.add_argument('-rc', '--readOnlyCache', help='Opens the cache file in read-only mode and disables updates to it. This dramatically decreases the memory used by the cache file. Default=Read and write to the cache file.', action='store_true') @@ -213,7 +218,7 @@ outputFileEncoding=commandLineArguments.outputFileEncoding promptFileEncoding=commandLineArguments.promptFileEncoding -parsingSettingsFileEncoding=commandLineArguments.parsingSettingsFileEncoding +#parsingSettingsFileEncoding=commandLineArguments.parsingSettingsFileEncoding languageCodesFileEncoding=commandLineArguments.languageCodesFileEncoding characterNamesDictionaryEncoding=commandLineArguments.characterNamesDictionaryEncoding @@ -419,15 +424,17 @@ mode='deepl_web' elif (translationEngine.lower()=='py3translationserver'): mode='py3translationserver' + implemented=True elif (translationEngine.lower()=='sugoi') : mode='sugoi' #Sugoi has a default port association and only supports Jpn->Eng translations, so having a dedicated entry for it is still useful for input validation, especially since it only supports a subset of the py3translationserver API. + implemented=True else: sys.exit(('\n Error. Invalid translation engine specified: "' + translationEngine + '"' + usageHelp).encode(consoleEncoding)) print(('Mode is set to: \''+str(mode)+'\'').encode(consoleEncoding)) if implemented == False: - sys.exit( ('\n\"'+mode+'\" not yet implemented. Please pick another translation engine. \n Translation engines: '+ translationEngines).encode(consoleEncoding) ) + sys.exit( ('\n\"'+mode+'\" not yet implemented. Please pick another translation engine. \n Translation engines: '+ translationEnginesAvailable).encode(consoleEncoding) ) # Certain files must always exist, like fileToTranslateFileName, and usually languageCodesFileName. @@ -482,7 +489,7 @@ def checkIfThisFolderExists(myFolder): fileToTranslateIsASpreadsheet=True else: fileToTranslateIsASpreadsheet=False - print( ('Error: Unrecognized extension for a spreadsheet: ' + str(fileToTranslateFileExtensionOnly)).encode(consoleEncoding) ) + print( ( 'Error: Unrecognized extension for a spreadsheet: ' + str(fileToTranslateFileExtensionOnly) ).encode(consoleEncoding) ) sys.exit(1) @@ -578,8 +585,9 @@ def checkIfThisFolderExists(myFolder): if outputFileName == None: # If no outputFileName was specified, then set it the same as the input file. This will have the date and appropriate extension appended to it later. - outputFileName=fileToTranslateFileName - #print('pie') + #outputFileName=fileToTranslateFileName + #Update: Just do it here instead. + outputFileName=fileToTranslateFileName + '.translated.' + py3TranslateLLMfunctions.getDateAndTimeFull() + defaultExportExtension outputFileNameWithoutPathOrExt=pathlib.Path(outputFileName).stem outputFileExtensionOnly=pathlib.Path(outputFileName).suffix @@ -644,7 +652,7 @@ def checkIfThisFolderExists(myFolder): #outputFileName -#outputFileEncoding= +outputFileEncoding=defaultTextEncoding #For the output file encoding: #if the user specified an input file, use the input file's encoding for the output file #if the user did not specify an input file, then the program cannot run, but they might have specified a spreadsheet (.csv, .xlsx, .xls, .ods) @@ -707,11 +715,15 @@ def checkIfThisFolderExists(myFolder): #languageCodesSpreadsheet = languageCodesWorkbook.active # skip reading languageCodesFileName if mode is parseOnly. +# Update: parseOnly mode is not really supported anymore. It should probably be renamed to dryRun mode where the purpose is to check for parsing errors in everything, including the languageCodes.csv and cache.xlsx files. if mode != 'parseOnly': - languageCodesSpreadsheet=chocolate.Strawberry(languageCodesFileName,languageCodesEncoding,ignoreWhitespaceForCSV=True) + print('languageCodesFileName='+languageCodesFileName) + print('languageCodesEncoding='+languageCodesEncoding) + + languageCodesSpreadsheet=chocolate.Strawberry(myFileName=languageCodesFileName,fileEncoding=languageCodesEncoding,removeWhitespaceForCSV=True) - #replace this code with dedicated search functions from chocolate - #use....chocolate.searchColumnsCaseInsensitive(spreadsheet, searchTerm) + # replace this code with dedicated search functions from chocolate.Strawberry() + # use....Strawberry().searchColumnsCaseInsensitive(spreadsheet, searchTerm) #languageCodesSpreadsheet #Source language is optional. @@ -784,35 +796,17 @@ def checkIfThisFolderExists(myFolder): # Next turn the main inputFile into a data structure. -#then create data structure seperately from reading the file +# then create data structure seperately from reading the file # This returns a very special dictionary where the value in key=value is a special list and then add data row by row using the dictionary values #Edit, moved to chocolate.py so as to not have to do that. All spreadsheets that require a parseFile will therefore always be Strawberries from the chocolate library. # Strawberry is a wrapper class for the workbook class with additional methods. # The interface has no concept of workbooks vs spreadsheets. That distinction is handled only inside the class. Syntax: # mainSpreadsheet=chocolate.Strawberry() # py3TranslateLLMfunctions.parseRawInputTextFile -# if the main file is a .txt, .ks or .ts file, then it will have a parse file. Otherwise, if it is a spreadsheet, then it will not have a parse file. -# if it is a spreadsheet file, then read it in as a datastructure natively. -if fileToTranslateIsASpreadsheet == True: - #then create data structure using that spreadsheet file. - mainSpreadsheet=chocolate.Strawberry( fileToTranslateFileName, fileToTranslateEncoding) -elif fileToTranslateIsASpreadsheet != True: - #This must have a parse file, so turn it into a dictionary. - #read in parseFile which returns as a dictionary. Parse file is usually needed because it has both parse settings (start of processing) and wordWrap settings (end of processing). However, neither parsing nor wordWrap are always needed since user can specify a seperate outfile which just dumps mainSpreadsheet with the translated values. - - parseSettingsDictionary = py3TranslateLLMfunctions.readSettingsFromTextFile(parseSettingsFileName, parseSettingsFileEncoding) - if debug == True: - print( ('parseSettingsDictionary='+str(parseSettingsDictionary)).encode(consoleEncoding) ) - - #And then use that dictionary to create a Strawberry() - #mainSpreadsheet = chocolate.Strawberry(fileToTranslateFileName, fileToTranslateEncoding, parseSettingsDict = parseSettingsDictionary, charaNamesDict = charaNamesDictionary) - - parsingScriptObject=pathlib.Path(parsingScript).absolute() - sys.path.append(str(parsingScriptObject.parent)) - parser=parsingScriptObject.name - import parser - - +# if main file is a spreadsheet, then it be read in as a native data structure. Otherwise, if the main file is a .txt file, then it will be parsed as line-by-line. +# Basically, the user is responsible for proper parsing if line-by-line parsing does not work right. Proper parsing is outside the scope of py3TranslateLLM +# Create data structure using fileToTranslateFileName. Whether it is a text file or spreadsheet file is handled internally. +mainSpreadsheet=chocolate.Strawberry( fileToTranslateFileName, fileToTranslateEncoding, addHeaderToTextFile=False) #Before doing anything, just blindly create a backup. #backupsFolder does not have / at the end @@ -833,6 +827,11 @@ def checkIfThisFolderExists(myFolder): #Now that the main data structure has been created, the spreadsheet is ready to be translated. if mode == 'parseOnly': + mainSpreadsheet.export(outputFileName,fileEncoding=outputFileEncoding,columnToExportForTextFiles='A') + #work complete. Exit. + sys.exit( 'Work complete.'.encode(consoleEncoding) ) + + # Old code. #The outputFileName will match fileToTranslateFileName if an output file name was not specified. If one was specified, then assume it had an extension. if outputFileName == fileToTranslateFileName: mainSpreadsheet.exportToXLSX( outputFileName + '.raw.' + py3TranslateLLMfunctions.getDateAndTimeFull() + '.xlsx' ) @@ -847,16 +846,15 @@ def checkIfThisFolderExists(myFolder): elif outputFileExtensionOnly == '.ods': mainSpreadsheet.exportToODS(outputFileName) elif outputFileExtensionOnly == '.txt': - mainSpreadsheet.exportToTextFile(outputFileName,'rawText') - #work complete. Exit. - sys.exit( 'Work complete.'.encode(consoleEncoding) ) + mainSpreadsheet.exportToTextFile(outputFileName,'A') + #Now need to translate stuff. -# Cache should always be added. This potentially that creates a situation where cache is not valid when going from one title to another or where it is used for translating entries for one character that another character spoke, but that is fine since that is a user decision. +# Cache should always be added. This potentially creates a situation where cache is not valid when going from one title to another or where it is used for translating entries for one character that another character spoke, but that is fine since that is a user decision to keep cache enabled despite the slight collisions. #First, initialize cache.xlsx file under backups/ -#Has same structure as mainSpreadsheet except for no speaker and no metadata. Multiple columns with each one. +# Has same structure as mainSpreadsheet except for no speaker and no metadata. Still has a header of course. Multiple columns with each one as a different translation engine. #if the path for cache does not exist, then create it. pathlib.Path( cachePathOnly ).mkdir( parents = True, exist_ok = True ) @@ -868,31 +866,106 @@ def checkIfThisFolderExists(myFolder): #Initalize Strawberry(). Very tempting to hardcode utf-8 here, but... will avoid. cache=chocolate.Strawberry(myFileName=cacheFileName,fileEncoding=defaultTextEncoding,createNew=True) #Since the Strawberry is brand new, add header row. - cache.appendRow(['rawText']) + cache.appendRow( ['rawText'] ) cache.exportToXLSX(cacheFileName) if (verbose == True) or (debug == True): cache.printAllTheThings() -#mainSpreadsheet=chocolate.Strawberry(fileToTranslateFileName, fileToTranslateEncoding) cacheFileName -# Implement KoboldAPI first, then DeepL, then Sugoi. +# Implement KoboldAPI first, then DeepL, . +# Update: Implement py3translationserver, then Sugoi, then KoboldCPP's API then DeepL API, then DeepL Web, then OpenAI's API. +# Check current engine. + # Echo request? Some firewalls block echo requests. + # Maybe just assume it exists and poke it with various requests until it is obvious to the user that it is not responding? + +# py3translationServer must be reachable Check by getting currently loaded model. This is required for the cache and mainSpreadsheet. +if mode == 'py3translationserver': + translationEngine=translationEngines.Py3translationServerEngine(sourceLanguage=sourceLanguageFullRow, targetLanguage=targetLanguageFullRow, address=address, port=port) + #Check if the server is reachable. If not, then exit. How? The py3translationServer should have both the version and model available at http://localhost:14366/api/v1/model and version, and should have been set during initalization, so verify they are not None. + + if translationEngine.model == None: + print( 'translationEngine.model is None' ) + sys.exit(1) + elif translationEngine.version == None: + print( 'translationEngine.version is None' ) + sys.exit(1) + + +# SugoiNMT server must be reachable. + # KoboldAPI must be reachable. Check by getting currently loaded model. This is required for the cache and mainSpreadsheet. #if not exist model in main spreadsheet, #then add it to headers and return current column for model. #else, return current column for model. + +# DeepL has already been imported, and it must have an API key. (already checked for) + #Must have internet access then. How to check? + # Added py3TranslateLLMfunctions.checkIfInternetIsAvailable() function. + + +if translationEngine.reachable != True: + print( 'TranslationEngine \''+ mode +'\' is not reachable. Check the connection settings and try again.' ) + sys.exit(1) + + +# This will return the column letter of the model if the model is already in the spreadsheet. Otherwise, if it is not found, then it will return None. +currentModelColumn = mainSpreadsheet.searchHeaders(translationEngine.model) +if currentModelColumn == None: + # Then the model is not currently in the spreadsheet, so need to add it. Update currentModelColumn after it has been updated. + headers = mainSpreadsheet.getRow( 1 ) + headers.append( translationEngine.model ) + mainSpreadsheet.replaceRow( 1, headers ) + currentModelColumn = mainSpreadsheet.searchHeaders(translationEngine.model) + if currentModelColumn == None: + print( 'unspecified error.' ) + sys.exit(1) + + # Check cache as well. #if not exist model in cache, #then add it to headers and return the cache's column for model. #else, return the cache's column for model. -# DeepL has already been imported, and it must have an API key. (already checked for) - #Must have internet access then. How to check? - # Added py3TranslateLLMfunctions.checkIfInternetIsAvailable() function. -# Check current engine. Sugoi/NMT server must be reachable. - # Echo request? Some firewalls block echo requests. - # Maybe just assume it exists and poke it with various requests until it is obvious to the user that it is not responding? +currentCacheColumn = cache.searchHeaders(translationEngine.model) +if currentCacheColumn == None: + # Then the model is not currently in the cache, so need to add it. Update currentCacheColumn after it has been updated. + headers = cache.getRow(1) + headers.append(translationEngine.model) + cache.replaceRow( 1, headers ) + currentCacheColumn = cache.searchHeaders(translationEngine.model) + if currentCacheColumn == None: + print( 'unspecified error .' ) + sys.exit(1) + + +if translationEngine.supportsBatches == True: + #translationEngine.batchTranslate() + # if there is a limit to how large a batch can be, then the server should handle that internally. + #currentModelColumn + untranslatedEntriesColumnFull=mainSpreadsheet.getColumn('A') + #tempHeader = untranslatedEntriesColumnFull[0] # Save the header. #Update: Wrong header. This is always 'rawText'. + if debug==True: + print( ( 'untranslatedEntriesColumnFull=' + str(untranslatedEntriesColumnFull) ).encode(consoleEncoding) ) + untranslatedEntriesColumnFull.pop(0) #This removes the header and returns the header. + translatedEntries = translationEngine.batchTranslate( untranslatedEntriesColumnFull ) + if debug==True: + print( ( 'translatedEntries=' + str(translatedEntries) ).encode(consoleEncoding) ) + + #translatedEntries and untranslatedEntriesColumnFull need to be added to the cache file now. + counter=0 +# for rawTextEntry in untranslatedEntriesColumnFull: +# if cache. rawTextEntry +#cache.searchFirstColumn('searchTerm') #can be used to check only the first column. Returns either None if not found or currentRow number if it was found. + translatedEntries.insert(0,translationEngine.model) # Put header back. This returns None. + mainSpreadsheet.replaceColumn( currentModelColumn , translatedEntries) + +else: + #translationEngine.translate() + pass +mainSpreadsheet.export(outputFileName,fileEncoding=outputFileEncoding,columnToExportForTextFiles=currentModelColumn) +#mainSpreadsheet.printAllTheThings() #Now have two column letters for both currentModelColumn and currentCacheColumn. #currentCacheColumn can be None if cache is disabled. cache might also be set to read only mode. diff --git a/resources/chocolate.py b/resources/chocolate.py index ff07686..c5bcf09 100644 --- a/resources/chocolate.py +++ b/resources/chocolate.py @@ -8,7 +8,7 @@ License: See main program. """ -__version__='2024Mar17' +__version__='2024Mar18' #set defaults printStuff=True @@ -56,7 +56,7 @@ class Strawberry: # self is not a keyword. It can be anything, like pie, but it must be the first argument for every function in the class. # Quirk: It can be different string/word for each method and they all still refer to the same object. - def __init__(self, myFileName=None, fileEncoding=None, ignoreWhitespaceForCSV=False,charaNamesDict=None,createNew=False): + def __init__(self, myFileName=None, fileEncoding=defaultTextFileEncoding, removeWhitespaceForCSV=False,createNew=False, addHeaderToTextFile=False): self.workbook = openpyxl.Workbook() self.spreadsheet = self.workbook.active @@ -80,9 +80,10 @@ def __init__(self, myFileName=None, fileEncoding=None, ignoreWhitespaceForCSV=Fa if os.path.isfile(myFileName) != True: sys.exit(('Error: This file does not exist:\''+myFileName+'\'').encode(consoleEncoding)) - #If extension = .csv, then call importFromCSV(myFileName) + # if extension = .csv, then call importFromCSV(myFileName) if myFileExtensionOnly == '.csv': - self.importFromCSV(myFileName, fileEncoding, ignoreWhitespaceForCSV) +# def importFromCSV(self, fileNameWithPath,myFileNameEncoding,errors=inputErrorHandling,removeWhitespaceForCSV=True ): + self.importFromCSV(myFileName, myFileNameEncoding=fileEncoding, removeWhitespaceForCSV=removeWhitespaceForCSV) #if extension = .xlsx, then call importFromXLSX(myFileName) elif myFileExtensionOnly == '.xlsx': self.importFromXLSX(myFileName, fileEncoding) @@ -91,15 +92,11 @@ def __init__(self, myFileName=None, fileEncoding=None, ignoreWhitespaceForCSV=Fa elif myFileExtensionOnly == '.ods': self.importFromODS(myFileName, fileEncoding) else: - #Else the file must be a text file to instantiate a class with. However, a parse file is required for that. - if parseSettingsDict == None: - sys.exit(('Warning: Cannot instantiate chocolate.Strawberry() using file with unknown extension:\'' + myFileExtensionOnly + '\' and no parseSettingsDictionary. Reference:\'' + myFileName + '\'').encode(consoleEncoding)) - else: - #print('pie') - #Since the Strawberry is being initalized from raw text data, add header row. - #Strawberries imported from external sources or from spreadsheets have the first row reserved for headers, so it should already be present - self.appendRow( ['rawText', 'speaker', 'metadata'] ) - self.importFromTextFile( myFileName, fileEncoding, parseSettingsDict, charaNamesDict) + #Else the file must be a text file to instantiate a class with. Only line-by-line parsing is supported. + print( ('Warning: Attempting to instantiate chocolate.Strawberry() using file with unknown extension:\'' + myFileExtensionOnly + '\' Reading in line-by-line. This is probably incorrect. Reference:\'' + myFileName + '\'').encode(consoleEncoding)) + if addHeaderToTextFile == True: + self.appendRow( ['rawText'] ) + self.importFromTextFile( myFileName, fileEncoding) def __str__(self): #maybe return the headers from the spreadsheet? @@ -121,9 +118,9 @@ def setCellValue(self, cellAddress,value): def getCellValue(self, cellAddress): return self.spreadsheet[cellAddress].value - # Full name of this function is getCellAddressFromRawCellString, but was shortened for legibility. Edit: Made it longer again. + # Full name of this function is _getCellAddressFromRawCellString, but was shortened for legibility. Edit: Made it longer again. # This functions would return 'B5' from: - def getCellAddressFromRawCellString(self, myInputCellRaw): + def _getCellAddressFromRawCellString(self, myInputCellRaw): #print('raw cell data='+str(myInputCellRaw)) #myInputCellRaw=str(myInputCellRaw) #Basically, split the string according to . and then split it again according to > to get back only the CellAddress @@ -132,11 +129,11 @@ def getCellAddressFromRawCellString(self, myInputCellRaw): # This function returns a list containing 2 strings that represent a row and column extracted from input Cell address # such as returning ['5', 'B'] from: It also works for complicated cases like AB534. - def getRowAndColumnFromRawCellString(self, myInputCellRaw): + def _getRowAndColumnFromRawCellString(self, myInputCellRaw): #print('raw cell data='+str(myInputCellRaw)) #basically, split the string according to . and then split it again according to > to get back only the CellAddress #myInputCell=str(myInputCellRaw).split('.', maxsplit=1)[1].split('>')[0] - myInputCell=self.getCellAddressFromRawCellString(myInputCellRaw) + myInputCell=self._getCellAddressFromRawCellString(myInputCellRaw) index=0 for i in range(10): #Magic number. try: @@ -162,7 +159,7 @@ def getRowAndColumnFromRawCellString(self, myInputCellRaw): # if i.value == 'lots of pies': # print(str(i) + '=' + str(i.value)) # myRawCell=i - #currentRow, currentColumn = spreadsheet.getRowAndColumnFromRawCellString(myRawCell) + #currentRow, currentColumn = spreadsheet._getRowAndColumnFromRawCellString(myRawCell) # Returns a list with the contents of the row number specified. @@ -174,8 +171,8 @@ def getRow(self, rowNumber): myList=[] for cell in self.spreadsheet[rowNumber]: if debug == True: - print( (str(self.spreadsheet[self.getCellAddressFromRawCellString(cell)].value)+',').encode(consoleEncoding),end='') - myList.append(self.spreadsheet[self.getCellAddressFromRawCellString(cell)].value) + print( (str(self.spreadsheet[self._getCellAddressFromRawCellString(cell)].value)+',').encode(consoleEncoding),end='') + myList.append(self.spreadsheet[self._getCellAddressFromRawCellString(cell)].value) if debug == True: print('') return myList @@ -186,9 +183,9 @@ def getRow(self, rowNumber): def getColumn(self, columnLetter): myList=[] for cell in self.spreadsheet[columnLetter]: - #print(str(mySpreadsheet[self.getCellAddressFromRawCellString(cell)].value)+',',end='') - #myList[i]=mySpreadsheet[self.getCellAddressFromRawCellString(cell)].value #Doesn't work due to out of index error. Use append() method. - myList.append(self.spreadsheet[self.getCellAddressFromRawCellString(cell)].value) + #print(str(mySpreadsheet[self._getCellAddressFromRawCellString(cell)].value)+',',end='') + #myList[i]=mySpreadsheet[self._getCellAddressFromRawCellString(cell)].value #Doesn't work due to out of index error. Use append() method. + myList.append(self.spreadsheet[self._getCellAddressFromRawCellString(cell)].value) return myList #print("Hello, world!") @@ -201,24 +198,28 @@ def getColumn(self, columnLetter): #Example: newRow = ['pies', 'lots of pies'] #mySpreadsheet.append(newRow) #The rowLocation specified is the nth rowLocation, not the [0,1,2,3...row] because rows start with 1 - def replaceRow(self, newRowList, rowLocation): + def replaceRow( self, rowLocation, newRowList ): if debug == True: - print(str(len(newRowList)).encode(consoleEncoding)) - print(str(range(len(newRowList))).encode(consoleEncoding)) + print( str(len(newRowList) ).encode(consoleEncoding)) + print( str(range(len(newRowList)) ).encode(consoleEncoding)) + print( ('newRowList=' + str(newRowList) ).encode(consoleEncoding) ) + for i in range(len(newRowList)): #Syntax for assignment is: mySpreadsheet['A4'] = 'pie' #mySpreadsheet['A4'] without an assignment returns: #columns begin with 1 instead of 0, so add 1 when referencing the target column, but not the source because source is a python list which are referenced as list[0], list[1], list[2], list[3], etc + #Was workaround for Syntax error cannot assign value to function call: mySpreadsheet.cell(row=5, column=3)='pies' - #spreadsheet[getCellAddressFromRawCellString(spreadsheet.cell(row=int(rowLocation), column=i+1))]=newRowList[i] + #spreadsheet[_getCellAddressFromRawCellString(spreadsheet.cell(row=int(rowLocation), column=i+1))]=newRowList[i] + #A more direct way of doing the same thing is to use .value without () on the cell after the cell reference. self.spreadsheet.cell(row=int(rowLocation), column=i+1).value=newRowList[i] #return myWorkbook - #Example: replaceRow(newRow,7) + #Example: replaceRow(7,newRow) - def replaceColumn(self, newColumnInAList, columnLetter): + def replaceColumn( self, columnLetter, newColumnInAList ): #So, how to convert a columnLetter into a number or does column='A' also work? #Answer column='A' does not work but there are some built in methods: #x = openpyxl.utils.column_index_from_string('A') #returns 1 as an int @@ -237,8 +238,16 @@ def replaceColumn(self, newColumnInAList, columnLetter): # Return either None if there is no cell with the search term, or the column letter of the cell if it found it. Case and whitespace sensitive search. - # Aside: To determine the row, the column, or both from the raw cell address, call self.getRowAndColumnFromRawCellString(rawCellAddress) + # Aside: To determine the row, the column, or both from the raw cell address, call self._getRowAndColumnFromRawCellString(rawCellAddress) def searchHeaders(self, searchTerm): + for row in self.spreadsheet.iter_rows(): + for cell in row: + if cell.value == searchTerm: + return self._getRowAndColumnFromRawCellString(cell)[1] + break + return None + + # Old code. cellFound=None for row in self.spreadsheet[1]: for i in row: @@ -254,9 +263,9 @@ def searchHeaders(self, searchTerm): return None #Slower. #else: - #myRowNumber, myColumnLetter = self.getRowAndColumnFromRawCellString(cellFound) + #myRowNumber, myColumnLetter = self._getRowAndColumnFromRawCellString(cellFound) #return myColumnLetter - return self.getRowAndColumnFromRawCellString(cellFound)[1] #Faster. + return self._getRowAndColumnFromRawCellString(cellFound)[1] #Faster. #Example: #cellFound=None @@ -270,7 +279,15 @@ def searchHeaders(self, searchTerm): # This searches the first column for the searchTerm and returns None if not found or the row number if it found it. # Case and whitespace sensitive search. def searchFirstColumn(self, searchTerm): - print('Hello, World!'.encode(consoleEncoding)) + #print('Hello, World!'.encode(consoleEncoding)) + for column in self.spreadsheet.iter_cols(): + for cell in column: + if cell.value == searchTerm: + return self._getRowAndColumnFromRawCellString(cell)[0] + break + return None + + # Old code. cellFound=None for column in self.spreadsheet['A']: #does this work? TODO: Test this. for i in column: @@ -280,35 +297,36 @@ def searchFirstColumn(self, searchTerm): break #stop searching after first column #Hummmm. if cellFound == None: return None - return self.getRowAndColumnFromRawCellString(cellFound)[0] + return self._getRowAndColumnFromRawCellString(cellFound)[0] # This returns either [None, None] if there is no cell with the search term, or a list containing the [row, column], the address. Case and whitespace sensitive. - #To determine the row, the column, or both from the raw cell address, use self.getRowAndColumnFromRawCellString(rawCellAddress) + #To determine the row, the column, or both from the raw cell address, use self._getRowAndColumnFromRawCellString(rawCellAddress) def searchSpreadsheet(self, searchTerm): for row in self.spreadsheet.iter_rows(): for cell in row: if cell.value == searchTerm: - return self.getRowAndColumnFromRawCellString(cell) + return self._getRowAndColumnFromRawCellString(cell) return [None, None] # These return either [None,None] if there is no cell with the search term, or a [list] containing the cell row and the cell column (the address in a list). Case insensitive. Whitespace sensitive. - # To determine the row, the column, or both from the raw cell address, use self.getRowAndColumnFromRawCellString(rawCellAddress) + # To determine the row, the column, or both from the raw cell address, use self._getRowAndColumnFromRawCellString(rawCellAddress) def searchRowsCaseInsensitive(self, searchTerm): for row in self.spreadsheet.iter_rows(): for cell in row: if isinstance( cell.value, (str, int) ): if cell.value.lower() == str(searchTerm).lower(): - return self.getRowAndColumnFromRawCellString(cell) + return self._getRowAndColumnFromRawCellString(cell) return [None, None] + def searchColumnsCaseInsensitive(self, searchTerm): for column in self.spreadsheet.iter_cols(): for cell in column: if isinstance( cell.value, (str, int) ): if cell.value.lower() == str(searchTerm).lower(): - return self.getRowAndColumnFromRawCellString(cell) + return self._getRowAndColumnFromRawCellString(cell) return [None, None] @@ -326,11 +344,38 @@ def printAllTheThings(self): #mySpreadsheet.printAllTheThings() + # Export spreadsheet to file, write it to the file system, based upon constructor settings, path, and file extension in the path. + def export(self, outputFileNameWithPath=None, fileEncoding=defaultTextFileEncoding, columnToExportForTextFiles='A'): + outputFileNameOnly, outputFileExtensionOnly = os.path.splitext( str(outputFileNameWithPath) ) + if outputFileExtensionOnly == '.csv': + #Should probably try to handle the path in a sane way. + self.exportToCSV(outputFileNameWithPath, fileEncoding=self.fileEncoding) + elif outputFileExtensionOnly == '.xlsx': + self.exportToXLSX(outputFileNameWithPath) + elif outputFileExtensionOnly == '.xls': + self.exportToXLS(outputFileNameWithPath) + elif outputFileExtensionOnly == '.ods': + self.exportToODS(outputFileNameWithPath) + elif outputFileExtensionOnly == '.txt': + self.exportToTextFile(outputFileNameWithPath, columnToExport=columnToExportForTextFiles, fileEncoding=self.fileEncoding) + else: + print( ( 'Warning: Unable to export chocolate.Strawberry() to file with unknown extension of \''+ outputFileExtensionOnly + '\' Full path: '+ str(outputFileNameWithPath) ).encode(consoleEncoding) ) + + + # Supports line by line parsing only. Header should already be part of text file. + def importFromTextFile(self, fileNameWithPath,fileEncoding=defaultTextFileEncoding): + myFileContents=[] + # Open file as text file with specified encoding and input error handler. + with open( fileNameWithPath, 'r', newline='', encoding=fileEncoding, errors=inputErrorHandling ) as myFileHandle: + # Create a list from every line and append that list to the current spreadsheet. + self.appendRow( [ myFileHandle.readline() ] ) + + #columnToExport to export can be a string or an int. if string, then represents name of column. If int, represents the column in the Strawberry() data structure. The int must be converted to a letter before exporting it. #if columnToExport == None: then dynamically calculate what should be exported. Only the translated line furthest to the right is valid to export, along with any untranslated lines. - #Honestly, exporting to text files does not really make sense unless line-by-line mode was enabled. Maybe remove all \n's from the output then? The translated lines should not have them, so just do not reinsert them and remove them from the source untranslated lines of there is no translated line for that row. - #When is this useful? What is the use case? - def exportToTextFile(self, fileNameWithPath,columnToExport=None): + # Honestly, exporting to text files does not really make sense unless line-by-line mode was enabled. Maybe remove all \n's from the output then? The translated lines should not have them, so just do not reinsert them and remove them from the source untranslated lines of there is no translated line for that row. + # When is this useful? What is the use case? It always makes more sense to export as .csv right? Otherwise, a specific column will need to be chosen and that should probably be exposed in the CLI. Otherwise, should a mixed mode be supported? Like exporting the right-most entry in the spreadsheet data structure? + def exportToTextFile(self, fileNameWithPath, columnToExport=None, fileEncoding=defaultTextFileEncoding): print('Hello World'.encode(consoleEncoding)) #print( ('Wrote: '+fileNameWithPath).encode(consoleEncoding) ) @@ -342,7 +387,8 @@ def exportToTextFile(self, fileNameWithPath,columnToExport=None): #Edit: Return value/reference for reading from files should be done by returning a class instance (object) of Strawberry() #Strawberry should have its own methods for writing to files of various formats. #All files follow the same rule of the first row being reserved for header values and invalid for inputting/outputting actual data. - def importFromCSV(self, fileNameWithPath,myFileNameEncoding,ignoreWhitespace=True): + def importFromCSV(self, fileNameWithPath,myFileNameEncoding=defaultTextFileEncoding,errors=inputErrorHandling,removeWhitespaceForCSV=True ): + print( ('Reading from: '+fileNameWithPath).encode(consoleEncoding) ) #import languageCodes.csv, but first check to see if it exists if os.path.isfile(fileNameWithPath) != True: sys.exit(('\n Error. Unable to find .csv file:"' + fileNameWithPath + '"').encode(consoleEncoding)) @@ -351,24 +397,24 @@ def importFromCSV(self, fileNameWithPath,myFileNameEncoding,ignoreWhitespace=Tru #tempSpreadsheet = tempWorkbook.active #tempSpreadsheet = Strawberry() - #It looks like quoting fields in csv's that use commas , and new - #lines works but only as double quotes " and not single quotes ' - #Spaces are also preserved as-is if they are within the commas (,) by default, so remove them - #If spaces are intended to be within the entry, then the user can encapslate them in double quotes - #Need to test. Even double quotes might not preserve them. Tested: They do not. - #Could also just say not supported since it is almost certainly an error for hand-written CSV's. - #Could also have a flag that switches back and forth. - #Partial solution, added "ignoreWhitespace" function parameter which defaults to True. - #Reading from dictionaries can be called with the "False" option for maximum flexibility. - #New problem: How to expose this functionality to user? Partial solution. Just use sensible defaults and have users fix their input. + # It looks like quoting fields in csv's that use commas , and new + # lines works but only as double quotes " and not single quotes ' + # Spaces are also preserved as-is if they are within the commas (,) by default, so remove them + # If spaces are intended to be within the entry, then the user can encapslate them in double quotes + # Need to test. Even double quotes might not preserve them. Tested: They do not. + # Could also just say not supported since it is almost certainly an error for hand-written CSV's. + # Could also have a flag that switches back and forth. + # Partial solution, added "removeWhitespaceForCSV" function parameter which defaults to True. + # Reading from dictionaries can be called with the "False" option for maximum flexibility. + # New problem: How to expose this functionality to user? Partial solution. Just use sensible defaults and have users fix their input. #print(inputErrorHandling) - with open(fileNameWithPath, newline='', encoding=myFileNameEncoding, errors=inputErrorHandling) as myFile:#shouldn't this be codecs.open and with error handling options? codecs seems to be an alias or something? #Edit: Turns out codecs was a relic from python 2 days. Python 3 integrated all of that, so codecs.open is not needed at all anymore. + with open(fileNameWithPath, newline='', encoding=myFileNameEncoding, errors=errors) as myFile: #shouldn't this be codecs.open and with error handling options? codecs seems to be an alias or something? #Edit: Turns out codecs was a relic from python 2 days. Python 3 integrated all of that, so codecs.open is not needed at all anymore. csvReader = csv.reader(myFile) for line in csvReader: if debug == True: print(str(line).encode(consoleEncoding)) #clean up whitespace for entities - if ignoreWhitespace == True: + if removeWhitespaceForCSV == True: #Not entirely sure what this for loop does or why it is needed, but just leave it alone. Was probably a bug fix for something at some point. Maybe it removes whitespace from like... , Eng,... and so forth? for i in range(len(line)): line[i]=line[i].strip() @@ -382,7 +428,7 @@ def importFromCSV(self, fileNameWithPath,myFileNameEncoding,ignoreWhitespace=Tru def exportToCSV(self, fileNameWithPath, fileEncoding=defaultTextFileEncoding,errors=outputErrorHandling): #print('Hello World'.encode(consoleEncoding)) - with open(fileNameWithPath, 'w', newline='', encoding=fileEncoding) as myOutputFileHandle: + with open(fileNameWithPath, 'w', newline='', encoding=fileEncoding,errors=errors) as myOutputFileHandle: myCsvHandle = csv.writer(myOutputFileHandle) # Get every row for current spreadsheet. @@ -393,9 +439,11 @@ def exportToCSV(self, fileNameWithPath, fileEncoding=defaultTextFileEncoding,err for cell in row: tempList.append( str(cell) ) myCsvHandle.writerow(tempList) + print( ('Wrote: '+fileNameWithPath).encode(consoleEncoding) ) def importFromXLSX(self, fileNameWithPath, fileEncoding=defaultTextFileEncoding): + print( ('Reading from: '+fileNameWithPath).encode(consoleEncoding) ) self.workbook=openpyxl.load_workbook(filename = fileNameWithPath) self.spreadsheet=self.workbook.active @@ -409,14 +457,18 @@ def exportToXLSX(self, fileNameWithPath, fileEncoding=defaultTextFileEncoding): def importFromXLS(self, fileNameWithPath, fileEncoding=defaultTextFileEncoding): print('Hello World'.encode(consoleEncoding)) + #print( ('Reading from: '+fileNameWithPath).encode(consoleEncoding) ) #return workbook + def exportToXLS(self, fileNameWithPath, fileEncoding=defaultTextFileEncoding): print('Hello World'.encode(consoleEncoding)) #print( ('Wrote: '+fileNameWithPath).encode(consoleEncoding) ) def importFromODS(self, fileNameWithPath, fileEncoding=defaultTextFileEncoding): print('Hello World'.encode(consoleEncoding)) + #print( ('Reading from: '+fileNameWithPath).encode(consoleEncoding) ) #return workbook + def exportToODS(self, fileNameWithPath, fileEncoding=defaultTextFileEncoding): print('Hello World'.encode(consoleEncoding)) #print( ('Wrote: '+fileNameWithPath).encode(consoleEncoding) ) @@ -425,13 +477,6 @@ def exportToODS(self, fileNameWithPath, fileEncoding=defaultTextFileEncoding): - - - - - - - """ #Usage examples, assuming this library is in a subfolder named 'resources': defaultEncoding='utf-8' diff --git a/resources/translationEngines.py b/resources/translationEngines.py index 68d5493..2f8e61b 100644 --- a/resources/translationEngines.py +++ b/resources/translationEngines.py @@ -3,6 +3,9 @@ """ Description: This library defines various translation engines to use when translating text. The idea is to expose a semi-uniform interface. These libraries assume the data will be input as either a single string or as a batch. Batches are a single Python list where each entry is a string. +Update: It might be better to create a subfolder called translationEngines and then split each engine into its own file. Then it would be imported as: +import resources.translationEngines.py3translationServerEngine as py3translationServerEngine + Usage: See below. Like at the bottom. License: See main program. @@ -12,17 +15,65 @@ printStuff=True verbose=False debug=False -#debug=True consoleEncoding='utf-8' -linesThatBeginWithThisAreComments='#' -assignmentOperatorInSettingsFile='=' -inputErrorHandling='strict' -#outputErrorHandling='namereplace' import requests #wrapper class for spreadsheet data structure -class SugoiNMT: +class Py3translationServerEngine: + # Address is the protocol and the ip address or hostname of the target server. + # sourceLanguage and targetLanguage are lists that have the full language, the two letter language codes, the three letter language codes, and some meta information useful for other translation engines. + def __init__(self, sourceLanguage=None, targetLanguage=None, address=None, port=None): + self.sourceLanguage=sourceLanguage + self.targetLanguage=targetLanguage + self.supportsBatches=True + self.supportsHistory=False + self.supportsPrompt=False + self.requiresPrompt=False + self.address=address + self.port=port + self.addressFull=self.address + ':' + str(self.port) + + self.reachable=False + #Some sort of test to check if the server is reachable goes here. Maybe just try to get model/version and if they are turned, the server is declared reachable? + + self.model=None + self.version=None + print( 'Connecting to py3translationServer at ' + self.addressFull + ' ...' ) + if (self.address != None) and (self.port != None): + try: + self.model = requests.get( self.addressFull + '/api/v1/model', timeout=10 ).text + self.version = requests.get( self.addressFull + '/api/v1/version', timeout=10 ).text + except requests.exceptions.ConnectTimeout: + print( 'Unable to connect to py3translationServer. Please check the connection settings and try again.' ) + + if self.model != None: + self.reachable=True + + + # This expects a python list where every entry is. + def batchTranslate(self, untranslatedList): + + #debug=True + if debug == True: + print( ( 'untranslatedList=' + str(untranslatedList) ).encode(consoleEncoding) ) + translatedList = requests.post( self.addressFull, json = dict ([ ('content' , untranslatedList ), ('message' , 'translate sentences') ]) ).json() + if debug == True: + print( ( 'translatedList=' + str(translatedList) ).encode(consoleEncoding) ) + + # print(len(untranslatedList)) + # print(len(translatedList)) + assert( len(translatedList) == len(untranslatedList) ) + + return translatedList + + + # This expects a string to translate. + def translate(self, untranslatedString): + return str( self.batchTranslate( [untranslatedString] ) ) # Lazy. + + +class SugoiNMTEngine: # Address is the protocol and the ip address or hostname of the target server. # sourceLanguage and targetLanguage are lists that have the full language, the two letter language codes, the three letter language codes, and some meta information useful for other translation engines. def __init__(self, sourceLanguage=None, targetLanguage=None, address=None, port=None): @@ -44,7 +95,7 @@ def __init__(self, sourceLanguage=None, targetLanguage=None, address=None, port= except: pass - + """ @@ -53,11 +104,12 @@ def __init__(self, sourceLanguage=None, targetLanguage=None, address=None, port= import translationEngines -translationEngine=translationEngines.KoboldCpp( sourceLanguage, targetLanguage, address, port, prompt ) -translationEngine=translationEngines.DeepLAPIFree( sourceLanguage, targetLanguage, APIKey, deeplDictionary ) -translationEngine=translationEngines.DeepLAPIPro( sourceLanguage, targetLanguage, APIKey, deeplDictionary ) -translationEngine=translationEngines.DeepLWeb( sourceLanguage, targetLanguage ) -translationEngine=translationEngines.SugoiNMT( sourceLanguage, targetLanguage, address, port ) +translationEngine=translationEngines.KoboldCppEngine( sourceLanguage, targetLanguage, address, port, prompt ) +translationEngine=translationEngines.DeepLAPIFreeEngine( sourceLanguage, targetLanguage, APIKey, deeplDictionary ) +translationEngine=translationEngines.DeepLAPIProEngine( sourceLanguage, targetLanguage, APIKey, deeplDictionary ) +translationEngine=translationEngines.DeepLWebEngine( sourceLanguage, targetLanguage ) +translationEngine=translationEngines.SugoiNMTEngine( sourceLanguage, targetLanguage, address, port ) +translationEngine=translationEngines.Py3translationServerEngine( sourceLanguage, targetLanguage, address, port ) translationEngine.supportsBatches translationEngine.supportsHistory @@ -69,7 +121,7 @@ def __init__(self, sourceLanguage=None, targetLanguage=None, address=None, port= translationEngine.port translationEngine.apiKey translationEngine.translate(mystring) -translationEngine.translate_batch(myList) +translationEngine.translateBatch(myList) batchesAreAvailable=translationEngine.supportsBatches if batchesAreAvailable == True: