diff --git a/app/scripts/translatte/commands/clearServerStrings.ts b/app/scripts/translatte/commands/clearServerStrings.ts index 6df5d923b..d36892240 100644 --- a/app/scripts/translatte/commands/clearServerStrings.ts +++ b/app/scripts/translatte/commands/clearServerStrings.ts @@ -1,11 +1,11 @@ -import { listToGroupList } from "@togglecorp/fujs"; +import { isTruthyString, listToGroupList } from "@togglecorp/fujs"; import { fetchServerState, postLanguageStrings, writeFilePromisify } from "../utils"; async function clearServerStrings(apiUrl: string, authToken: string) { const serverStrings = await fetchServerState(apiUrl, authToken); const bulkActions = listToGroupList( - serverStrings, + serverStrings.filter(({ page_name }) => isTruthyString(page_name)), ({ language }) => language, ({ key, page_name }) => ({ action: "delete" as const, @@ -19,7 +19,7 @@ async function clearServerStrings(apiUrl: string, authToken: string) { response: object, }[] = []; - console.log('Pusing delete actions for en...') + console.log('Pushing delete actions for en...') const enResponse = await postLanguageStrings( 'en', bulkActions.en, @@ -31,7 +31,7 @@ async function clearServerStrings(apiUrl: string, authToken: string) { logs.push({ responseFor: 'en', response: enResponseJson }); - console.log('Pusing delete actions for fr...') + console.log('Pushing delete actions for fr...') const frResponse = await postLanguageStrings( 'fr', bulkActions.fr, @@ -42,7 +42,7 @@ async function clearServerStrings(apiUrl: string, authToken: string) { const frResponseJson = await frResponse.json(); logs.push({ responseFor: 'fr', response: frResponseJson }); - console.log('Pusing delete actions for es...') + console.log('Pushing delete actions for es...') const esResponse = await postLanguageStrings( 'es', bulkActions.es, @@ -52,7 +52,7 @@ async function clearServerStrings(apiUrl: string, authToken: string) { const esResponseJson = await esResponse.json(); logs.push({ responseFor: 'es', response: esResponseJson }); - console.log('Pusing delete actions for ar...') + console.log('Pushing delete actions for ar...') const arResponse = await postLanguageStrings( 'ar', bulkActions.ar, diff --git a/app/scripts/translatte/commands/exportServerStringsToExcel.ts b/app/scripts/translatte/commands/exportServerStringsToExcel.ts index bcfa7f8e9..73b8f966f 100644 --- a/app/scripts/translatte/commands/exportServerStringsToExcel.ts +++ b/app/scripts/translatte/commands/exportServerStringsToExcel.ts @@ -1,7 +1,7 @@ import xlsx from 'exceljs'; import { fetchServerState } from "../utils"; -import { isFalsyString, listToGroupList, listToMap, mapToList } from '@togglecorp/fujs'; +import { isFalsyString, isTruthyString, listToGroupList, listToMap, mapToList } from '@togglecorp/fujs'; async function exportServerStringsToExcel( apiUrl: string, @@ -35,7 +35,7 @@ async function exportServerStringsToExcel( const keyGroupedStrings = mapToList( listToGroupList( - serverStrings, + serverStrings.filter(({ page_name, key }) => isTruthyString(page_name) && isTruthyString(key)), ({ page_name, key }) => `${page_name}:${key}`, ), (list) => { diff --git a/app/scripts/translatte/commands/pushMigration.ts b/app/scripts/translatte/commands/pushMigration.ts index bce31ef3a..74969d1e9 100644 --- a/app/scripts/translatte/commands/pushMigration.ts +++ b/app/scripts/translatte/commands/pushMigration.ts @@ -102,6 +102,10 @@ async function pushMigration(migrationFilePath: string, apiUrl: string, authToke const serverActionsForCurrentLanguage = actions.flatMap((actionItem) => { if (language === 'en') { if (actionItem.action === 'add') { + if (isFalsyString(actionItem.value)) { + return undefined; + } + return { action: 'set' as const, key: actionItem.key, @@ -168,7 +172,7 @@ async function pushMigration(migrationFilePath: string, apiUrl: string, authToke ); await writeFilePromisify( - `server-actions.json`, + `/tmp/server-actions.json`, JSON.stringify(serverActions, null, 2), 'utf8', ); @@ -193,7 +197,7 @@ async function pushMigration(migrationFilePath: string, apiUrl: string, authToke const setActions = actions.filter(({ action }) => action === 'set'); const deleteActions = actions.filter(({ action }) => action === 'delete'); - console.log(`Pusing deleted actions for ${lang}...`) + console.log(`Pushing deleted actions for ${lang}...`) const deleteResponse = await postLanguageStrings( lang, deleteActions, @@ -221,7 +225,7 @@ async function pushMigration(migrationFilePath: string, apiUrl: string, authToke await applyAction(serverActions.ar.language, serverActions.ar.actions); await writeFilePromisify( - `push-migration-logs.json`, + `/tmp/push-migration-logs.json`, JSON.stringify(logs, null, 2), 'utf8', ); diff --git a/app/scripts/translatte/commands/pushStringsDref.ts b/app/scripts/translatte/commands/pushStringsDref.ts new file mode 100644 index 000000000..062ca3820 --- /dev/null +++ b/app/scripts/translatte/commands/pushStringsDref.ts @@ -0,0 +1,206 @@ +import xlsx, { CellValue } from 'exceljs'; +import { fetchServerState, postLanguageStrings } from "../utils"; +import { encodeDate, isDefined, isNotDefined, listToGroupList, listToMap, mapToList } from '@togglecorp/fujs'; +import { Language, ServerActionItem, SourceStringItem } from '../types'; +import { Md5 } from 'ts-md5'; + + +function getValueFromCellValue(cellValue: CellValue) { + if (isNotDefined(cellValue)) { + return undefined; + } + + if ( + typeof cellValue === 'number' + || typeof cellValue === 'string' + || typeof cellValue === 'boolean' + ) { + return cellValue; + } + + if (cellValue instanceof Date) { + return encodeDate(cellValue); + } + + if ('error' in cellValue) { + return undefined; + } + + if ('richText' in cellValue) { + return cellValue.richText.map(({ text }) => text).join(''); + } + + if ('hyperlink' in cellValue) { + const MAIL_IDENTIFIER = 'mailto:'; + if (cellValue.hyperlink.startsWith(MAIL_IDENTIFIER)) { + return cellValue.hyperlink.substring(MAIL_IDENTIFIER.length); + } + + return cellValue.hyperlink; + } + + if (isNotDefined(cellValue.result)) { + return undefined; + } + + if (typeof cellValue.result === 'object' && 'error' in cellValue.result) { + return undefined; + } + + // Formula result + return getValueFromCellValue(cellValue.result); +} + +async function pushStringsDref(importFilePath: string, apiUrl: string, accessToken: string) { + const strings = await fetchServerState(apiUrl); + const enStrings = strings.filter((string) => string.language === 'en'); + + const workbook = new xlsx.Workbook(); + + await workbook.xlsx.readFile(importFilePath); + + const firstSheet = workbook.worksheets[0]; + const columns = firstSheet.columns.map( + (column) => { + const key = column.values?.[1]?.toString(); + if (isNotDefined(key)) { + return undefined; + } + return { key, column: column.number } + } + ).filter(isDefined); + + const columnMap = listToMap( + columns, + ({ key }) => key, + ({ column }) => column, + ); + + const updatedStrings: SourceStringItem[] = []; + + firstSheet.eachRow((row) => { + const keyColumn = columnMap['Key']; + const key = isDefined(keyColumn) ? String(getValueFromCellValue(row.getCell(keyColumn).value)) : undefined; + + const namespaceColumn = columnMap['Namespace']; + const namespace = isDefined(namespaceColumn) ? String(getValueFromCellValue(row.getCell(namespaceColumn).value)) : undefined; + + if (isNotDefined(key) || isNotDefined(namespace)) { + return; + } + + const enColumnKey = columnMap['EN']; + const frColumnKey = columnMap['FR']; + const esColumnKey = columnMap['ES']; + const arColumnKey = columnMap['AR']; + + const enValue = isDefined(enColumnKey) ? getValueFromCellValue(row.getCell(enColumnKey).value) : undefined; + + const strings = enStrings.filter(({ value }) => value === enValue); + + if (strings.length > 0) { + strings.forEach((string) => { + const frValue = isDefined(frColumnKey) ? getValueFromCellValue(row.getCell(frColumnKey).value) : undefined; + const esValue = isDefined(esColumnKey) ? getValueFromCellValue(row.getCell(esColumnKey).value) : undefined; + const arValue = isDefined(arColumnKey) ? getValueFromCellValue(row.getCell(arColumnKey).value) : undefined; + + updatedStrings.push({ + ...string, + language: 'fr', + value: String(frValue), + }); + + updatedStrings.push({ + ...string, + language: 'es', + value: String(esValue), + }); + + updatedStrings.push({ + ...string, + language: 'ar', + value: String(arValue), + }); + }); + } + + if (strings.length === 0) { + const frValue = isDefined(frColumnKey) ? getValueFromCellValue(row.getCell(frColumnKey).value) : undefined; + const esValue = isDefined(esColumnKey) ? getValueFromCellValue(row.getCell(esColumnKey).value) : undefined; + const arValue = isDefined(arColumnKey) ? getValueFromCellValue(row.getCell(arColumnKey).value) : undefined; + + const hash = Md5.hashStr(String(enValue)); + + updatedStrings.push({ + key, + page_name: namespace, + hash, + language: 'en', + value: String(enValue), + }); + + updatedStrings.push({ + key, + page_name: namespace, + hash, + language: 'fr', + value: String(frValue), + }); + + updatedStrings.push({ + key, + page_name: namespace, + hash, + language: 'es', + value: String(esValue), + }); + + updatedStrings.push({ + key, + page_name: namespace, + hash, + language: 'ar', + value: String(arValue), + }); + } + }); + + const languageGroupedActions = mapToList( + listToGroupList( + updatedStrings, + ({ language }) => language, + (languageString) => { + const serverAction: ServerActionItem = { + action: 'set', + key: languageString.key, + page_name: languageString.page_name, + value: languageString.value, + hash: languageString.hash, + } + + return serverAction; + }, + ), + (actions, language) => ({ + language: language as Language, + actions, + }) + ); + + for (let i = 0; i < languageGroupedActions.length; i++) { + const action = languageGroupedActions[i]; + + console.log(`posting ${action.language} actions...`); + const result = await postLanguageStrings( + action.language, + action.actions, + apiUrl, + accessToken, + ) + + const resultJson = await result.json(); + console.info(resultJson); + } +} + +export default pushStringsDref; diff --git a/app/scripts/translatte/commands/pushStringsFromExcel.ts b/app/scripts/translatte/commands/pushStringsFromExcel.ts index b9da7b38b..b0ed5ccab 100644 --- a/app/scripts/translatte/commands/pushStringsFromExcel.ts +++ b/app/scripts/translatte/commands/pushStringsFromExcel.ts @@ -1,9 +1,55 @@ -import xlsx from 'exceljs'; +import xlsx, { CellValue } from 'exceljs'; import { Md5 } from 'ts-md5'; -import { isDefined, isNotDefined, listToGroupList, listToMap, mapToList } from '@togglecorp/fujs'; +import { encodeDate, isDefined, isFalsyString, isNotDefined, isTruthyString, listToGroupList, listToMap, mapToList } from '@togglecorp/fujs'; import { Language, ServerActionItem } from '../types'; -import { postLanguageStrings } from '../utils'; +import { postLanguageStrings, writeFilePromisify } from '../utils'; + +function getValueFromCellValue(cellValue: CellValue) { + if (isNotDefined(cellValue)) { + return undefined; + } + + if ( + typeof cellValue === 'number' + || typeof cellValue === 'string' + || typeof cellValue === 'boolean' + ) { + return cellValue; + } + + if (cellValue instanceof Date) { + return encodeDate(cellValue); + } + + if ('error' in cellValue) { + return undefined; + } + + if ('richText' in cellValue) { + return cellValue.richText.map(({ text }) => text).join(''); + } + + if ('hyperlink' in cellValue) { + const MAIL_IDENTIFIER = 'mailto:'; + if (cellValue.hyperlink.startsWith(MAIL_IDENTIFIER)) { + return cellValue.hyperlink.substring(MAIL_IDENTIFIER.length); + } + + return cellValue.hyperlink; + } + + if (isNotDefined(cellValue.result)) { + return undefined; + } + + if (typeof cellValue.result === 'object' && 'error' in cellValue.result) { + return undefined; + } + + // Formula result + return getValueFromCellValue(cellValue.result); +} async function pushStringsFromExcel(importFilePath: string, apiUrl: string, accessToken: string) { const workbook = new xlsx.Workbook(); @@ -36,28 +82,36 @@ async function pushStringsFromExcel(importFilePath: string, apiUrl: string, acce }[] = []; firstSheet.eachRow( - (row) => { - const keyColumn = columnMap['key']; - const key = isDefined(keyColumn) ? row.getCell(keyColumn).value?.toString() : undefined; + (row, i) => { + if (i === 0) { + return; + } - const namespaceColumn = columnMap['namespace']; - const namespace = isDefined(namespaceColumn) ? row.getCell(namespaceColumn).value?.toString() : undefined; + const keyColumn = columnMap['Key']; + const key = isDefined(keyColumn) ? String(getValueFromCellValue(row.getCell(keyColumn).value)) : undefined; - if (isNotDefined(key) || isNotDefined(namespace)) { + const namespaceColumn = columnMap['Namespace']; + const namespace = isDefined(namespaceColumn) ? String(getValueFromCellValue(row.getCell(namespaceColumn).value)) : undefined; + + if (isFalsyString(key) || isFalsyString(namespace)) { return; } - const enColumn = columnMap['en']; - const en = isDefined(enColumn) ? row.getCell(enColumn).value?.toString() : undefined; + const enColumn = columnMap['EN']; + const en = isDefined(enColumn) ? String(getValueFromCellValue(row.getCell(enColumn).value)) : undefined; + + if (isFalsyString(en)) { + return; + } - const arColumn = columnMap['ar']; - const ar = isDefined(arColumn) ? row.getCell(arColumn).value?.toString() : undefined; + const arColumn = columnMap['AR']; + const ar = isDefined(arColumn) ? String(getValueFromCellValue(row.getCell(arColumn).value)) : undefined; - const frColumn = columnMap['fr']; - const fr = isDefined(frColumn) ? row.getCell(frColumn).value?.toString() : undefined; + const frColumn = columnMap['FR']; + const fr = isDefined(frColumn) ? String(getValueFromCellValue(row.getCell(frColumn).value)) : undefined; - const esColumn = columnMap['es']; - const es = isDefined(esColumn) ? row.getCell(esColumn).value?.toString() : undefined; + const esColumn = columnMap['ES']; + const es = isDefined(esColumn) ? String(getValueFromCellValue(row.getCell(esColumn).value)) : undefined; if (isNotDefined(en)) { return; @@ -73,7 +127,7 @@ async function pushStringsFromExcel(importFilePath: string, apiUrl: string, acce hash, }); - if (isDefined(ar)) { + if (isTruthyString(ar)) { strings.push({ key, namespace, @@ -83,7 +137,7 @@ async function pushStringsFromExcel(importFilePath: string, apiUrl: string, acce }); } - if (isDefined(fr)) { + if (isTruthyString(fr)) { strings.push({ key, namespace, @@ -93,7 +147,7 @@ async function pushStringsFromExcel(importFilePath: string, apiUrl: string, acce }); } - if (isDefined(es)) { + if (isTruthyString(es)) { strings.push({ key, namespace, @@ -127,16 +181,26 @@ async function pushStringsFromExcel(importFilePath: string, apiUrl: string, acce }) ); - const postPromises = languageGroupedActions.map( - (languageStrings) => postLanguageStrings( - languageStrings.language, - languageStrings.actions, + await writeFilePromisify( + '/tmp/language-grouped-actions.json', + JSON.stringify(languageGroupedActions, null, 2), + 'utf8', + ); + + for (let i = 0; i < languageGroupedActions.length; i++) { + const action = languageGroupedActions[i]; + + console.log(`posting ${action.language} actions...`); + const result = await postLanguageStrings( + action.language, + action.actions, apiUrl, accessToken, - ) - ) + ); - await Promise.all(postPromises); + const resultJson = await result.text(); + console.info(resultJson); + } } export default pushStringsFromExcel; diff --git a/app/scripts/translatte/commands/syncEnStrings.ts b/app/scripts/translatte/commands/syncEnStrings.ts new file mode 100644 index 000000000..3b3b7982e --- /dev/null +++ b/app/scripts/translatte/commands/syncEnStrings.ts @@ -0,0 +1,39 @@ +import { isDefined, isFalsyString } from "@togglecorp/fujs"; +import { fetchServerState, postLanguageStrings, writeFilePromisify } from "../utils"; + +async function syncEnStrings(sourceApiUrl: string, desinationApiUrl: string, authToken: string) { + const serverStrings = await fetchServerState(sourceApiUrl, authToken); + const enStrings = serverStrings.filter((string) => string.language === 'en'); + + const actions = enStrings.map((string) => { + if (isFalsyString(string.key) || isFalsyString(string.page_name) || isFalsyString(string.value)) { + return undefined; + } + + return { + action: 'set' as const, + key: string.key, + page_name: string.page_name, + value: string.value, + hash: string.hash, + }; + }).filter(isDefined); + + console.log("posting en actions..."); + const result = await postLanguageStrings( + 'en', + actions, + desinationApiUrl, + authToken, + ) + + const resultJson = await result.json(); + console.info(resultJson); + await writeFilePromisify( + '/tmp/sync-en-strings-logs.json', + JSON.stringify(resultJson, null, 2), + 'utf8', + ); +} + +export default syncEnStrings; diff --git a/app/scripts/translatte/main.ts b/app/scripts/translatte/main.ts index f8dd65a23..9a2f34c02 100644 --- a/app/scripts/translatte/main.ts +++ b/app/scripts/translatte/main.ts @@ -14,6 +14,8 @@ import pushMigration from './commands/pushMigration'; import pushStringsFromExcel from './commands/pushStringsFromExcel'; import exportServerStringsToExcel from './commands/exportServerStringsToExcel'; import clearServerStrings from './commands/clearServerStrings'; +import pushStringsDref from './commands/pushStringsDref'; +import syncEnStrings from './commands/syncEnStrings'; const currentDir = cwd(); @@ -254,6 +256,67 @@ yargs(hideBin(process.argv)) ); }, ) + .command( + 'push-strings-dref ', + 'IMPORTANT!!! Temporary command, do not use!', + (yargs) => { + yargs.positional('IMPORT_FILE_PATH', { + type: 'string', + describe: 'Find the import file on IMPORT_FILE_PATH', + }); + yargs.options({ + 'auth-token': { + type: 'string', + describe: 'Authentication token to access the API server', + require: true, + }, + 'api-url': { + type: 'string', + describe: 'URL for the API server', + require: true, + } + }); + }, + async (argv) => { + const importFilePath = (argv.IMPORT_FILE_PATH as string); + + await pushStringsDref( + importFilePath, + argv.apiUrl as string, + argv.authToken as string, + ); + }, + ) + .command( + 'sync-en-strings', + 'IMPORTANT!!! Temporary command, do not use!', + (yargs) => { + yargs.options({ + 'auth-token': { + type: 'string', + describe: 'Authentication token to access the API server', + require: true, + }, + 'source-api-url': { + type: 'string', + describe: 'URL for the source API server', + require: true, + }, + 'destination-api-url': { + type: 'string', + describe: 'URL for the destination', + require: true, + } + }); + }, + async (argv) => { + await syncEnStrings( + argv.sourceApiUrl as string, + argv.destinationApiUrl as string, + argv.authToken as string, + ); + }, + ) .command( 'export-server-strings ', 'Export server strings to excel file', diff --git a/go-api b/go-api index 596b1f2c2..14be734d9 160000 --- a/go-api +++ b/go-api @@ -1 +1 @@ -Subproject commit 596b1f2c24014148879a97cc069a62205dfdc24d +Subproject commit 14be734d91ca8e26e2727a9814935a85750d51ec