[Feature] Bulk Transaction Import via CSV and JSON with Validation Engine #578

controllers/importController.js

@@ -0,0 +1,146 @@
+const Transaction = require('../models/Transaction');
+const User = require('../models/User');
+const xlsx = require('exceljs');
+const Joi = require('joi');
+const { Readable } = require('stream');
+
+const transactionSchema = Joi.object({
+    description: Joi.string().trim().max(100).required(),
+    amount: Joi.number().min(0.01).required(),
+    currency: Joi.string().uppercase().length(3).optional(),
+    category: Joi.string().valid('food', 'transport', 'entertainment', 'utilities', 'healthcare', 'shopping', 'other', 'salary', 'freelance', 'investment', 'transfer').required(),
+    type: Joi.string().valid('income', 'expense', 'transfer').required(),
+    merchant: Joi.string().trim().max(50).optional(),
+    date: Joi.date().optional()
+});
+
+exports.importTransactions = async (req, res) => {
+    if (!req.file) {
+        return res.status(400).json({ error: 'No file uploaded' });
+    }
+
+    // req.file.buffer contains the data because middleware uses memoryStorage
+    const fileExtension = req.file.originalname.split('.').pop().toLowerCase();
+    let transactions = [];
+    let errors = [];
+    let importedCount = 0;
+    let skippedCount = 0;
+
+    try {
+        // Parse File
+        if (fileExtension === 'json') {
+            try {
+                const jsonString = req.file.buffer.toString('utf8');
+                transactions = JSON.parse(jsonString);
+            } catch (e) {
+                return res.status(400).json({ error: 'Invalid JSON format' });
+            }
+        } else if (fileExtension === 'csv') {
+            const workbook = new xlsx.Workbook();
+            const stream = Readable.from(req.file.buffer);
+            await workbook.csv.read(stream);
+
+            const worksheet = workbook.getWorksheet(1);
+            if (!worksheet) {
+                return res.status(400).json({ error: 'Invalid CSV file' });
+            }
+
+            // Assuming first row is header
+            const headers = [];
+            worksheet.getRow(1).eachCell((cell, colNumber) => {
+                headers[colNumber] = cell.value ? cell.value.toString().toLowerCase().trim() : '';
+            });
+
+            worksheet.eachRow((row, rowNumber) => {
+                if (rowNumber === 1) return; // Skip header
+                const transaction = {};
+                row.eachCell((cell, colNumber) => {
+                    const header = headers[colNumber];
+                    if (header) {
+                        // cell.value can be an object in exceljs if it's a formula or rich text, but for CSV usually simple
+                        transaction[header] = (cell.value && typeof cell.value === 'object' && cell.value.result) ? cell.value.result : cell.value;
+                    }
+                });
+                if (Object.keys(transaction).length > 0) {
+                    transactions.push(transaction);
+                }
+            });
+        } else {
+            return res.status(400).json({ error: 'Unsupported file format. Please upload CSV or JSON.' });
+        }
+
+        // Process Transactions
+        const user = await User.findById(req.user._id);
+
+        for (const rawData of transactions) {
+            let amount = rawData.amount;
+            // Handle "1,000.00" string amounts
+            if (typeof amount === 'string') {
+                amount = parseFloat(amount.replace(/,/g, ''));
+            }
+
+            let date = rawData.date ? new Date(rawData.date) : new Date();
+
+            const transactionData = {
+                description: rawData.description || 'Imported Transaction',
+                amount: amount,
+                currency: rawData.currency || user.preferredCurrency || 'INR',
+                category: rawData.category ? rawData.category.toLowerCase() : 'other',
+                type: rawData.type ? rawData.type.toLowerCase() : 'expense',
+                merchant: rawData.merchant || '',
+                date: date
+            };
+
+            // Validate
+            const { error, value } = transactionSchema.validate(transactionData);
+            if (error) {
+                errors.push({
+                    transaction: rawData,
+                    error: error.details[0].message
+                });
+                continue;
+            }
+
+            // Check Duplicate
+            const startWindow = new Date(value.date);
+            startWindow.setSeconds(startWindow.getSeconds() - 60);
+            const endWindow = new Date(value.date);
+            endWindow.setSeconds(endWindow.getSeconds() + 60);
+
+            const existing = await Transaction.findOne({
+                user: req.user._id,
+                amount: value.amount,
+                description: value.description,
+                type: value.type,
+                date: { $gte: startWindow, $lte: endWindow }
+            });
+
+            if (existing) {
+                skippedCount++;
+                continue;
+            }
+
+            // Create Transaction
+            await Transaction.create({
+                ...value,
+                user: req.user._id,
+                originalAmount: value.amount,
+                originalCurrency: value.currency,
+                kind: value.type
+            });
+            importedCount++;
+        }
+
+        res.json({
+            success: true,
+            imported: importedCount,
+            skipped: skippedCount,
+            errors: errors.length > 0 ? errors : undefined,
+            message: `Successfully imported ${importedCount} transactions. ${skippedCount} skipped as duplicates.`
+        });
+
+    } catch (error) {
+        console.error('Import error:', error);
+        res.status(500).json({ error: 'Failed to process import file: ' + error.message });
+    }
+};

public/js/import-modal.js

@@ -0,0 +1,173 @@
+class ImportModalManager {
+    constructor() {
+        this.file = null;
+        this.init();
+    }
+
+    init() {
+        // We will bind events after the modal HTML is injected or we can rely on onclick attributes
+    }
+
+    openModal() {
+        const modal = document.getElementById('importModal');
+        if (modal) {
+            modal.style.display = 'block';
+            this.resetForm();
+        }
+    }
+
+    closeModal() {
+        const modal = document.getElementById('importModal');
+        if (modal) {
+            modal.style.display = 'none';
+            this.resetForm();
+        }
+    }
+
+    resetForm() {
+        this.file = null;
+        const fileInput = document.getElementById('importFile');
+        if (fileInput) fileInput.value = '';
+
+        const preview = document.getElementById('filePreview');
+        if (preview) {
+            preview.innerHTML = '';
+            preview.style.display = 'none';
+        }
+
+        const error = document.getElementById('importError');
+        if (error) {
+            error.style.display = 'none';
+            error.textContent = '';
+        }
+
+        const success = document.getElementById('importSuccess');
+        if (success) {
+            success.style.display = 'none';
+            success.textContent = '';
+        }
+    }
+
+    handleFileSelect(event) {
+        const file = event.target.files[0];
+        if (!file) return;
+
+        // Validate file type
+        const validTypes = ['.csv', '.json', 'application/json', 'text/csv', 'application/vnd.ms-excel'];
+        const fileExtension = '.' + file.name.split('.').pop().toLowerCase();
+
+        // Simple client-side validation
+        if (file.size > 10 * 1024 * 1024) {
+            this.showError('File size exceeds 10MB limit.');
+            this.resetForm();
+            return;
+        }
+
+        this.file = file;
+        this.showPreview(file);
+        this.showError(''); // Clear error
+    }
+
+    showPreview(file) {
+        const preview = document.getElementById('filePreview');
+        if (preview) {
+            preview.innerHTML = `
+                <div class="file-info">
+                    <i class="fas ${file.name.endsWith('.csv') ? 'fa-file-csv' : 'fa-file-code'}"></i>
+                    <span>${file.name}</span>
+                    <small>(${(file.size / 1024).toFixed(2)} KB)</small>
+                </div>
+            `;
+            preview.style.display = 'block';
+        }
+    }
+
+    showError(message) {
+        const error = document.getElementById('importError');
+        if (error) {
+            error.textContent = message;
+            error.style.display = message ? 'block' : 'none';
+        }
+    }
+
+    showSuccess(message) {
+        const success = document.getElementById('importSuccess');
+        if (success) {
+            success.textContent = message;
+            success.style.display = message ? 'block' : 'none';
+        }
+    }
+
+    async uploadFile() {
+        if (!this.file) {
+            this.showError('Please select a file to upload.');
+            return;
+        }
+
+        const formData = new FormData();
+        formData.append('receipt', this.file); // Using 'receipt' as field name to match uploadMiddleware config
+
+        const submitBtn = document.getElementById('btnImportSubmit');
+        if (submitBtn) {
+            submitBtn.disabled = true;
+            submitBtn.innerHTML = '<i class="fas fa-spinner fa-spin"></i> Importing...';
+        }
+
+        try {
+            const response = await fetch('/api/expenses/import', {
+                method: 'POST',
+                headers: {
+                    'Authorization': `Bearer ${localStorage.getItem('token')}`
+                },
+                body: formData
+            });
+
+            const data = await response.json();
+
+            if (response.ok) {
+                this.showSuccess(data.message);
+                this.showError('');
+
+                // Show imported/skipped stats
+                setTimeout(() => {
+                    this.closeModal();
+                    if (window.transactionsManager) {
+                        window.transactionsManager.loadTransactions();
+                        window.transactionsManager.showNotification(data.message, 'success');
+                    } else {
+                        location.reload();
+                    }
+                }, 2000); // Wait 2s to show success message in modal
+            } else {
+                throw new Error(data.error || 'Upload failed');
+            }
+        } catch (error) {
+            console.error('Upload Error:', error);
+            this.showError(error.message);
+        } finally {
+            if (submitBtn) {
+                submitBtn.disabled = false;
+                submitBtn.innerHTML = '<i class="fas fa-file-import"></i> Import Transactions';
+            }
+        }
+    }
+}
+
+const importModalManager = new ImportModalManager();
+
+// Expose functions globally for onclick handlers
+function openImportModal() {
+    importModalManager.openModal();
+}
+
+function closeImportModal() {
+    importModalManager.closeModal();
+}
+
+function handleImportFileSelect(event) {
+    importModalManager.handleFileSelect(event);
+}
+
+function submitImport() {
+    importModalManager.uploadFile();
+}

public/transactions.html

@@ -34,6 +34,10 @@
             <div class="header-content">
                 <h1>💳 Transaction History</h1>
                 <div class="header-actions">
+                    <button class="import-btn" onclick="openImportModal()"
+                        style="background-color: #2563eb; color: white; padding: 0.5rem 1rem; border-radius: 6px; border: none; cursor: pointer; display: flex; align-items: center; gap: 0.5rem; margin-right: 0.5rem;">
+                        <i class="fas fa-file-import"></i> Import
+                    </button>
                     <button class="add-btn" onclick="openAddModal()">
                         <i class="fas fa-plus"></i> Add Transaction
                     </button>
@@ -327,9 +331,43 @@ <h3>Bulk Categorize</h3>
                 </div>
             </div>
         </div>
-    </div>
-    <script src="protect.js"></script>
-    <script src="transactions.js"></script>
+        <!-- Import Modal -->
+        <div id="importModal" class="modal">
+            <div class="modal-content">
+                <div class="modal-header">
+                    <h3>Import Transactions</h3>
+                    <button class="modal-close" onclick="closeImportModal()">
+                        <i class="fas fa-times"></i>
+                    </button>
+                </div>
+                <div class="import-body">
+                    <p>Upload a CSV or JSON file to import transactions.</p>
+                    <div class="file-upload-container" onclick="document.getElementById('importFile').click()">
+                        <i class="fas fa-cloud-upload-alt"></i>
+                        <p>Click to upload or drag and drop</p>
+                        <input type="file" id="importFile"
+                            accept=".csv,.json,application/json,text/csv,application/vnd.ms-excel"
+                            style="display: none;" onchange="handleImportFileSelect(event)">
+                    </div>
+                    <div id="filePreview" class="file-preview" style="display: none;"></div>
+                    <div id="importError" class="error-message" style="display: none;"></div>
+                    <div id="importSuccess" class="success-message" style="display: none;"></div>
+                    <div class="template-download">
+                        <small>Need a template? <a href="/templates/transactions.csv" download>Download CSV
+                                Template</a></small>
+                    </div>
+                </div>
+                <div class="modal-actions">
+                    <button class="btn-cancel" onclick="closeImportModal()">Cancel</button>
+                    <button id="btnImportSubmit" class="btn-submit" onclick="submitImport()">
+                        <i class="fas fa-file-import"></i> Import Transactions
+                    </button>
+                </div>
+            </div>
+        </div>
+        <script src="js/import-modal.js"></script>
+        <script src="protect.js"></script>
+        <script src="transactions.js"></script>
 </body>
 
 </html>

routes/expenses.js

@@ -11,7 +11,7 @@ const { asyncHandler } = require('../middleware/errorMiddleware');
 const { ExpenseSchemas, validateRequest, validateQuery } = require('../middleware/inputValidator');
 const { expenseLimiter, exportLimiter } = require('../middleware/rateLimiter');
 const { NotFoundError } = require('../utils/AppError');
-const {requireAuth,getUserId}=require('../middleware/clerkAuth');
+const { requireAuth, getUserId } = require('../middleware/clerkAuth');
 
 
 const router = express.Router();
@@ -297,4 +297,7 @@ router.post('/report/preview', requireAuth, asyncHandler(async (req, res) => {
   return ResponseFactory.success(res, previewData);
 }));
 
+// Import transactions
+router.post('/import', requireAuth, require('../middleware/uploadMiddleware').upload, require('../controllers/importController').importTransactions);
+
 module.exports = router;