diff --git a/package-lock.json b/package-lock.json index 3f02893..74a4190 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "gas-fire", - "version": "2.0.0", + "version": "1.1.3", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "gas-fire", - "version": "2.0.0", + "version": "1.1.3", "license": "MIT", "workspaces": [ "client", diff --git a/package.json b/package.json index 4b94208..d44d440 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "gas-fire", - "version": "1.1.1", + "version": "1.1.3", "author": "Melle Dijkstra", "description": "Google App Script utilities to automate the FIRE google sheet", "license": "MIT", diff --git a/src/server/category_detection.ts b/src/server/category-detection/index.ts similarity index 100% rename from src/server/category_detection.ts rename to src/server/category-detection/index.ts diff --git a/src/server/config.ts b/src/server/config.ts index d7112ad..fcaef2b 100644 --- a/src/server/config.ts +++ b/src/server/config.ts @@ -1,4 +1,4 @@ -import { buildColumn } from './table-utils'; +import { TableUtils, buildColumn } from './table-utils'; import { Transformers } from './transformers'; import type { Strategy, StrategyOption, Table } from './types'; import { n26Cols, raboCols, openbankCols } from './types'; @@ -12,93 +12,95 @@ export const AUTO_FILL_COLUMNS = [ ]; const defaultAfterImport = [ - (table: Table) => Utils.autoFillColumns(table, AUTO_FILL_COLUMNS), + (table: Table) => TableUtils.autoFillColumns(table, AUTO_FILL_COLUMNS), ]; type RootConfig = { [key in StrategyOption]: Strategy; }; +const n26Config: Strategy = { + beforeImport: [ + TableUtils.deleteLastRow, + TableUtils.deleteFirstRow, + TableUtils.sortByDate(n26Cols.Date), + ], + columnImportRules: { + ref: null, + iban: (data) => new Array(data.length).fill(Utils.getBankIban('N26')), + date: buildColumn(n26Cols.Date, (val) => new Date(val)), + amount: buildColumn(n26Cols.Amount, parseFloat), + category: buildColumn(n26Cols.Payee, Transformers.transformCategory), + contra_account: buildColumn(n26Cols.Payee, String), + label: buildColumn(n26Cols.TransactionType, String), + import_date: (data) => new Array(data.length).fill(new Date()), + description: buildColumn(n26Cols.PaymentReference, String), + contra_iban: buildColumn(n26Cols.AccountNumber, String), + currency: buildColumn(n26Cols.ForeignCurrencyType, String), + }, + afterImport: defaultAfterImport, +}; + +const rabobankConfig: Strategy = { + beforeImport: [ + TableUtils.deleteLastRow, + TableUtils.deleteFirstRow, + TableUtils.sortByDate(raboCols.Datum), + ], + columnImportRules: { + ref: buildColumn(raboCols.Volgnr, parseInt), + iban: buildColumn(raboCols.Iban, String), + date: buildColumn(raboCols.Datum, (val) => new Date(val)), + amount: buildColumn(raboCols.Bedrag, Transformers.transformMoney), + category: null, + contra_account: buildColumn(raboCols.NaamTegenpartij, String), + import_date: (data) => new Array(data.length).fill(new Date()), + contra_iban: buildColumn(raboCols.Tegenrekening, String), + currency: buildColumn(raboCols.Munt, String), + description: buildColumn(raboCols.Omschrijving1, String), + label: buildColumn(raboCols.Omschrijving2, String), + }, + afterImport: defaultAfterImport, +}; + +const openbankConfig: Strategy = { + beforeImport: [ + TableUtils.deleteFirstRow, + TableUtils.deleteLastRow, + // open bank has some empty columns when importing + (table) => TableUtils.deleteColumns(table, [0, 2, 4, 6, 8]), + ], + columnImportRules: { + ref: null, + iban: (data) => new Array(data.length).fill(Utils.getBankIban('OPENBANK')), + date: buildColumn(openbankCols.Fecha, (val) => { + let [day, month, year] = val.split('/'); + let yearNum = +year; + if (year && year.length === 2) { + // if year is of length 2 it means it only provides the year since 2000 + // to fix we add 2000 + yearNum = +year + 2000; + } + return new Date(+yearNum, +month - 1, +day); + }), + amount: buildColumn(openbankCols.Importe, Transformers.transformMoney), + category: null, + contra_account: null, + label: null, + description: buildColumn(openbankCols.Concepto, String), + import_date: (data) => new Array(data.length).fill(new Date()), + contra_iban: null, + currency: null, + }, + afterImport: defaultAfterImport, +}; + export class Config { static getConfig(): RootConfig { return { - n26: { - beforeImport: [ - Utils.deleteLastRow, - Utils.deleteFirstRow, - Utils.sortByDate(n26Cols.Date), - ], - columnImportRules: { - ref: null, - iban: (data) => new Array(data.length).fill(Utils.getBankIban('N26')), - date: buildColumn(n26Cols.Date, (val) => new Date(val)), - amount: buildColumn(n26Cols.Amount, parseFloat), - category: buildColumn(n26Cols.Payee, Transformers.transformCategory), - contra_account: buildColumn(n26Cols.Payee, String), - label: buildColumn(n26Cols.TransactionType, String), - import_date: (data) => new Array(data.length).fill(new Date()), - description: buildColumn(n26Cols.PaymentReference, String), - contra_iban: buildColumn(n26Cols.AccountNumber, String), - currency: buildColumn(n26Cols.ForeignCurrencyType, String), - }, - afterImport: defaultAfterImport, - }, - rabobank: { - beforeImport: [ - Utils.deleteLastRow, - Utils.deleteFirstRow, - Utils.sortByDate(raboCols.Datum), - ], - columnImportRules: { - ref: buildColumn(raboCols.Volgnr, parseInt), - iban: buildColumn(raboCols.Iban, String), - date: buildColumn(raboCols.Datum, (val) => new Date(val)), - amount: buildColumn(raboCols.Bedrag, Transformers.transformMoney), - category: null, - contra_account: buildColumn(raboCols.NaamTegenpartij, String), - import_date: (data) => new Array(data.length).fill(new Date()), - contra_iban: buildColumn(raboCols.Tegenrekening, String), - currency: buildColumn(raboCols.Munt, String), - description: buildColumn(raboCols.Omschrijving1, String), - label: buildColumn(raboCols.Omschrijving2, String), - }, - afterImport: defaultAfterImport, - }, - openbank: { - beforeImport: [ - Utils.deleteFirstRow, - Utils.deleteLastRow, - // open bank has some empty columns when importing - (table) => Utils.deleteColumns(table, [0, 2, 4, 6, 8]), - ], - columnImportRules: { - ref: null, - iban: (data) => - new Array(data.length).fill(Utils.getBankIban('OPENBANK')), - date: buildColumn(openbankCols.Fecha, (val) => { - let [day, month, year] = val.split('/'); - let yearNum = +year; - if (year && year.length === 2) { - // if year is of length 2 it means it only provides the year since 2000 - // to fix we add 2000 - yearNum = +year + 2000; - } - return new Date(+yearNum, +month - 1, +day); - }), - amount: buildColumn( - openbankCols.Importe, - Transformers.transformMoney - ), - category: null, - contra_account: null, - label: null, - description: buildColumn(openbankCols.Concepto, String), - import_date: (data) => new Array(data.length).fill(new Date()), - contra_iban: null, - currency: null, - }, - afterImport: defaultAfterImport, - }, + n26: n26Config, + rabobank: rabobankConfig, + openbank: openbankConfig, }; } } diff --git a/src/server/exposed_functions.ts b/src/server/exposed_functions.ts index 51732f5..545bdc0 100644 --- a/src/server/exposed_functions.ts +++ b/src/server/exposed_functions.ts @@ -4,7 +4,7 @@ * @param xpath simplified xpath as string * @customfunction */ -function IMPORTJSON(url: string, xpath: string) { +export function IMPORTJSON(url: string, xpath: string) { try { // /rates/EUR var res = UrlFetchApp.fetch(url); @@ -90,7 +90,7 @@ function IMPORTJSON(url: string, xpath: string) { * @return {string} The hashed input value. * @customfunction */ -function MD5(input: string): string { +export function MD5(input: string): string { var txtHash = ''; var rawHash = Utilities.computeDigest( Utilities.DigestAlgorithm.MD5, diff --git a/src/server/globals.ts b/src/server/globals.ts new file mode 100644 index 0000000..0cba223 --- /dev/null +++ b/src/server/globals.ts @@ -0,0 +1,24 @@ +import { getSheetById } from './utils'; + +export const fireColumns = [ + 'ref', + 'iban', + 'date', + 'amount', + 'balance', + 'contra_account', + 'description', + 'satisfaction', + 'icon', + 'category', + 'label', + 'hours', + 'contra_iban', + 'disabled', + 'currency', +]; + +export const SOURCE_SHEET_ID = 1093484485; +export const FireSpreadsheet = SpreadsheetApp.getActiveSpreadsheet(); +export const sheets = FireSpreadsheet.getSheets(); +export const sourceSheet = getSheetById(SOURCE_SHEET_ID); diff --git a/src/server/index.ts b/src/server/index.ts index d97f461..e485bd5 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -1,7 +1,8 @@ +// Functions to setup the initial UI export { onOpen, fileUploadDialog, openAboutDialog } from './ui'; -export { - generatePreview, - getStrategyOptions, - processCSV, -} from './remote-calls'; +// Remote procedure calls made by the client UI executed on the server +export { getStrategyOptions, processCSV } from './remote-calls'; + +// Custom functions that can be used within the Spreadsheet UI +export { IMPORTJSON, MD5 } from './exposed_functions'; diff --git a/src/server/remote-calls.ts b/src/server/remote-calls.ts index b88112f..0b2244d 100644 --- a/src/server/remote-calls.ts +++ b/src/server/remote-calls.ts @@ -1,10 +1,10 @@ +import { sourceSheet } from './globals'; import { Config } from './config'; -import { buildNewTableData } from './table-utils'; -import { ServerResponse, Strategy, StrategyOption, Table } from './types'; -import { Utils, sourceSheet } from './utils'; +import { TableUtils, processTableWithImportRules } from './table-utils'; +import { ServerResponse, StrategyOption, Table } from './types'; export function processCSV( - input: Table, + inputTable: Table, importStrategy: StrategyOption ): ServerResponse { const strategies = Config.getConfig(); @@ -19,12 +19,12 @@ export function processCSV( if (beforeImport) { for (const rule of beforeImport) { - input = rule(input); + inputTable = rule(inputTable); } } - let output = buildNewTableData(input, columnImportRules); - Utils.importData(output); + let output = processTableWithImportRules(inputTable, columnImportRules); + TableUtils.importData(output); if (afterImport) { for (const rule of afterImport) { @@ -45,10 +45,3 @@ export function processCSV( export function getStrategyOptions(): typeof StrategyOption { return StrategyOption; } - -export function generatePreview( - data: Table, - strategy: StrategyOption -): { result: Table; newBalance: number } { - return { result: data, newBalance: 1240.56 }; -} diff --git a/src/server/table-utils.ts b/src/server/table-utils.ts index 410d287..b879f2f 100644 --- a/src/server/table-utils.ts +++ b/src/server/table-utils.ts @@ -1,23 +1,5 @@ +import { fireColumns, sourceSheet } from './globals'; import { ColumnRule, FireColumnRules, InputColumn, Table } from './types'; -import { Utils } from './utils'; - -export const FireColumns = [ - 'ref', - 'iban', - 'date', - 'amount', - 'balance', - 'contra_account', - 'description', - 'satisfaction', - 'icon', - 'category', - 'label', - 'hours', - 'contra_iban', - 'disabled', - 'currency', -]; export function buildColumn( column: InputColumn, @@ -25,7 +7,7 @@ export function buildColumn( ): (data: Table) => T[] { return (data: Table): T[] => { const rowCount = data.length; - const columnTable = Utils.transpose(data); // try to transpose somewhere else + const columnTable = TableUtils.transpose(data); // try to transpose somewhere else if (columnTable[column] !== undefined) { return columnTable[column].map((val) => transformer(val)); } else { @@ -34,13 +16,19 @@ export function buildColumn( }; } -export function buildNewTableData( +/** + * What the heck does this function do? + * @param input + * @param columnImportRules + * @returns + */ +export function processTableWithImportRules( input: Table, columnImportRules: FireColumnRules ) { let output: Table = []; const rowCount = input.length; - for (const columnName of FireColumns) { + for (const columnName of fireColumns) { if (!(columnName in columnImportRules) || !columnImportRules[columnName]) { output.push(new Array(rowCount)); continue; @@ -49,13 +37,112 @@ export function buildNewTableData( let column: any[]; try { column = colRule(input); - column = Utils.ensureLength(column, rowCount); + column = TableUtils.ensureLength(column, rowCount); } catch (e) { Logger.log(e); column = new Array(rowCount); } output.push(column); } - output = Utils.transpose(output); // flip columns to rows + output = TableUtils.transpose(output); // flip columns to rows return output; } + +export class TableUtils { + /** + * Imports data in structure of a table into the source sheet + * @param {Table} data the data to be imported into the source sheet + */ + static importData(data: Table) { + const rowCount = data.length; + const colCount = data[0].length; + console.log(`importing data (rows: ${rowCount}, cols: ${colCount})`); + sourceSheet + ?.insertRowsBefore(2, rowCount) + .getRange(2, 1, rowCount, colCount) + .setValues(data as Table); + } + + /** + * @see https://github.com/ramda/ramda/blob/v0.27.0/source/transpose.js + */ + static transpose(outerlist: T[][]): T[][] { + let i = 0; + let result = []; + while (i < outerlist.length) { + let innerlist = outerlist[i]; + let j = 0; + while (j < innerlist.length) { + if (typeof result[j] === 'undefined') { + result[j] = []; + } + result[j].push(innerlist[j]); + j += 1; + } + i += 1; + } + return result; + } + + static deleteFirstRow(data: Table): Table { + data.shift(); + return data; + } + + static deleteLastRow(data: Table): Table { + data.pop(); + return data; + } + + static sortByDate(dateColumn: number) { + return (data: Table) => { + data + .sort( + (row1, row2) => + new Date(row1[dateColumn]).getUTCDate() - + new Date(row2[dateColumn]).getUTCDate() + ) + .reverse(); + return data; + }; + } + + static deleteColumns(table: Table, colIndices: number[]): Table { + // we want to have the indices sorted backwards to prevent shifting of elements + // while traversing the array + const sortedIndices = colIndices.sort().reverse(); + // tranpose the table so we are working with columns first instead of rows + let transposedTable = this.transpose(table); + Logger.log('transposed'); + Logger.log(transposedTable); + // delIndex is the column index to delete in the table + for (const delIndex of sortedIndices) { + if (typeof transposedTable[delIndex] !== 'undefined') { + transposedTable.splice(delIndex, 1); + } + } + // transpose again back to rows first + return this.transpose(transposedTable); + } + + static autoFillColumns(data: Table, columns: number[]) { + for (const column of columns) { + const rowCount = data.length; + const sourceRange = sourceSheet?.getRange(2 + rowCount, column); + const destinationRange = sourceSheet?.getRange(2, column, rowCount + 1); // + 1 because sourceRange needs to be included + if (destinationRange) { + sourceRange?.autoFill( + destinationRange, + SpreadsheetApp.AutoFillSeries.DEFAULT_SERIES + ); + } + } + } + + static ensureLength(arr: unknown[], length: number) { + if (arr.length < length) { + arr = arr.fill(null, arr.length, length - 1); + } + return arr; + } +} diff --git a/src/server/transformers.ts b/src/server/transformers.ts index 7bacd78..2eee30e 100644 --- a/src/server/transformers.ts +++ b/src/server/transformers.ts @@ -1,4 +1,4 @@ -import { detectCategoryByTextAnalysis } from './category_detection'; +import { detectCategoryByTextAnalysis } from './category-detection'; export class Transformers { static transformMoney(value: string, decimalSeparator: string = ','): number { diff --git a/src/server/utils.ts b/src/server/utils.ts index 54a83d6..cb87d0c 100644 --- a/src/server/utils.ts +++ b/src/server/utils.ts @@ -1,9 +1,4 @@ -import { Table, InputColumn } from './types'; - -export const SOURCE_SHEET_ID = 1093484485; - -const FireSpreadsheet = SpreadsheetApp.getActiveSpreadsheet(); -const sheets = FireSpreadsheet.getSheets(); +import { FireSpreadsheet, sheets } from './globals'; export function getSheetById( id: number @@ -11,102 +6,7 @@ export function getSheetById( return sheets.find((sheet) => sheet.getSheetId() === id); } -export const sourceSheet = getSheetById(SOURCE_SHEET_ID); - export class Utils { - /** - * @see https://github.com/ramda/ramda/blob/v0.27.0/source/transpose.js - */ - static transpose(outerlist: T[][]): T[][] { - let i = 0; - let result = []; - while (i < outerlist.length) { - let innerlist = outerlist[i]; - let j = 0; - while (j < innerlist.length) { - if (typeof result[j] === 'undefined') { - result[j] = []; - } - result[j].push(innerlist[j]); - j += 1; - } - i += 1; - } - return result; - } - - static importData(data: Table) { - const rowCount = data.length; - const colCount = data[0].length; - Logger.log(`importing data (rows: ${rowCount}, cols: ${colCount})`); - sourceSheet - ?.insertRowsBefore(2, rowCount) - .getRange(2, 1, rowCount, colCount) - .setValues(data as Table); - } - - static autoFillColumns(data: Table, columns: number[]) { - for (const column of columns) { - const rowCount = data.length; - const sourceRange = sourceSheet?.getRange(2 + rowCount, column); - const destinationRange = sourceSheet?.getRange(2, column, rowCount + 1); // + 1 because sourceRange needs to be included - if (destinationRange) { - sourceRange?.autoFill( - destinationRange, - SpreadsheetApp.AutoFillSeries.DEFAULT_SERIES - ); - } - } - } - - static ensureLength(arr: any[], length: number) { - if (arr.length < length) { - arr = arr.fill(null, arr.length, length - 1); - } - return arr; - } - - static deleteFirstRow(data: Table): Table { - data.shift(); - return data; - } - - static deleteLastRow(data: Table): Table { - data.pop(); - return data; - } - - static deleteColumns(table: Table, colIndices: number[]): Table { - // we want to have the indices sorted backwards to prevent shifting of elements - // while traversing the array - const sortedIndices = colIndices.sort().reverse(); - // tranpose the table so we are working with columns first instead of rows - let transposedTable = this.transpose(table); - Logger.log('transposed'); - Logger.log(transposedTable); - // delIndex is the column index to delete in the table - for (const delIndex of sortedIndices) { - if (typeof transposedTable[delIndex] !== 'undefined') { - transposedTable.splice(delIndex, 1); - } - } - // transpose again back to rows first - return this.transpose(transposedTable); - } - - static sortByDate(dateColumn: InputColumn) { - return (data: Table) => { - data - .sort( - (row1, row2) => - new Date(row1[dateColumn]).getUTCDate() - - new Date(row2[dateColumn]).getUTCDate() - ) - .reverse(); - return data; - }; - } - static getBankAccounts(): Record { // this range contains the ibans only const ibans = FireSpreadsheet.getRangeByName('accounts');