From e156d612fac23c87333d8930d3a786a5d720f7f4 Mon Sep 17 00:00:00 2001 From: Vishwa Karthik Date: Sun, 23 Feb 2025 10:59:09 +0530 Subject: [PATCH 1/3] enhance clearing history service via compute & isolate methods --- lib/consts.dart | 2 + lib/main.dart | 2 +- lib/services/history_service.dart | 136 ++++++++++++++++++++++++------ lib/services/hive_services.dart | 19 +++++ lib/utils/history_utils.dart | 42 ++++++++- 5 files changed, 173 insertions(+), 28 deletions(-) diff --git a/lib/consts.dart b/lib/consts.dart index 5937636e1..bde1f846a 100644 --- a/lib/consts.dart +++ b/lib/consts.dart @@ -70,6 +70,7 @@ final kIconRemoveLight = Icon( const kCodePreviewLinesLimit = 500; enum HistoryRetentionPeriod { + fiveSeconds("5 Seconds", Icons.access_time_rounded), oneWeek("1 Week", Icons.calendar_view_week_rounded), oneMonth("1 Month", Icons.calendar_view_month_rounded), threeMonths("3 Months", Icons.calendar_month_rounded), @@ -481,3 +482,4 @@ const kMsgClearHistory = const kMsgClearHistorySuccess = 'History cleared successfully'; const kMsgClearHistoryError = 'Error clearing history'; const kMsgShareError = "Unable to share"; +const int kIsolateThreshold = 50; diff --git a/lib/main.dart b/lib/main.dart index 8b5fab32a..edee291ae 100644 --- a/lib/main.dart +++ b/lib/main.dart @@ -47,7 +47,7 @@ Future initApp( ); debugPrint("openBoxesStatus: $openBoxesStatus"); if (openBoxesStatus) { - await autoClearHistory(settingsModel: settingsModel); + await HistoryServiceImpl().autoClearHistory(settingsModel: settingsModel); } return openBoxesStatus; } catch (e) { diff --git a/lib/services/history_service.dart b/lib/services/history_service.dart index 487b91640..f1ad5a184 100644 --- a/lib/services/history_service.dart +++ b/lib/services/history_service.dart @@ -1,42 +1,126 @@ +import 'dart:developer'; + +import 'dart:isolate'; +import 'dart:async'; +import 'package:hive_flutter/hive_flutter.dart'; +import 'package:apidash/consts.dart'; +import 'package:flutter/foundation.dart' show compute; import 'package:apidash/models/models.dart'; import 'package:apidash/utils/utils.dart'; +import 'package:flutter/material.dart' show debugPrint; + import 'hive_services.dart'; -Future autoClearHistory({SettingsModel? settingsModel}) async { - final historyRetentionPeriod = settingsModel?.historyRetentionPeriod; - DateTime? retentionDate = getRetentionDate(historyRetentionPeriod); +abstract class HistoryService { + Future autoClearHistory({required SettingsModel? settingsModel}); +} + +class HistoryServiceImpl implements HistoryService { + @override + Future autoClearHistory({required SettingsModel? settingsModel}) async { + try { + final historyRetentionPeriod = settingsModel?.historyRetentionPeriod; + DateTime? retentionDate = getRetentionDate(historyRetentionPeriod); + if (retentionDate == null) return; - if (retentionDate == null) { - return; - } else { - List? historyIds = hiveHandler.getHistoryIds(); - List toRemoveIds = []; + List? historyIds = hiveHandler.getHistoryIds(); + if (historyIds == null || historyIds.isEmpty) return; - if (historyIds == null || historyIds.isEmpty) { - return; - } + List toRemoveIds = historyIds.where((historyId) { + var jsonModel = hiveHandler.getHistoryMeta(historyId); + if (jsonModel != null) { + var jsonMap = Map.from(jsonModel); + HistoryMetaModel historyMetaModelFromJson = + HistoryMetaModel.fromJson(jsonMap); + return historyMetaModelFromJson.timeStamp.isBefore(retentionDate); + } + return false; + }).toList(); - for (var historyId in historyIds) { - var jsonModel = hiveHandler.getHistoryMeta(historyId); - if (jsonModel != null) { - var jsonMap = Map.from(jsonModel); - HistoryMetaModel historyMetaModelFromJson = - HistoryMetaModel.fromJson(jsonMap); - if (historyMetaModelFromJson.timeStamp.isBefore(retentionDate)) { - toRemoveIds.add(historyId); + if (toRemoveIds.isEmpty) return; + + int batchSize = calculateOptimalBatchSize(toRemoveIds.length); + final batches = createBatches(toRemoveIds, batchSize); + + final String hiveBoxPath = await getHiveBoxPath(); + + for (var batch in batches) { + if (toRemoveIds.length < kIsolateThreshold) { + await compute(_deleteRecordsCompute, + {"batch": batch, "hiveBoxPath": hiveBoxPath}); + } else { + await _deleteRecordsWithIsolate( + batch: batch, hiveBoxPath: hiveBoxPath); } } + + hiveHandler.setHistoryIds(historyIds..removeWhere(toRemoveIds.contains)); + } catch (e, st) { + log("Error clearing history records", + name: "autoClearHistory", error: e, stackTrace: st); } + } + + Future _deleteRecordsCompute(Map args) async { + List batch = args['batch'] as List; + String? hiveBoxPath = args['hiveBoxPath'] as String; - if (toRemoveIds.isEmpty) { - return; + try { + Hive.init(hiveBoxPath); + await openHiveBoxes(); + + for (var id in batch) { + hiveHandler.deleteHistoryMeta(id); + hiveHandler.deleteHistoryRequest(id); + } + } catch (e, st) { + log("Unable to open hive box", + name: "_deleteRecordsCompute", error: e, stackTrace: st); + } finally { + await closeHiveBoxes(); } + } - for (var id in toRemoveIds) { - await hiveHandler.deleteHistoryRequest(id); - hiveHandler.deleteHistoryMeta(id); + Future _deleteRecordsWithIsolate({ + required List batch, + required String? hiveBoxPath, + }) async { + final receivePort = ReceivePort(); + + try { + await Isolate.spawn( + _isolateTask, [receivePort.sendPort, batch, hiveBoxPath]); + + await receivePort.first; + } catch (e, st) { + debugPrint("Unable to instantiate Isolate"); + log("Unable to instantiate Isolate", + name: "_deleteRecordsWithIsolate", error: e, stackTrace: st); + } finally { + receivePort.close(); + } + } + + Future _isolateTask(List args) async { + SendPort sendPort = args[0] as SendPort; + List batch = args[1] as List; + String? hiveBoxPath = args[2] as String; + + Hive.init(hiveBoxPath); + await openHiveBoxes(); + + try { + for (var id in batch) { + hiveHandler.deleteHistoryMeta(id); + hiveHandler.deleteHistoryRequest(id); + } + sendPort.send(true); + } catch (e, st) { + sendPort.send(false); + log("Isolate task failed", + name: "_isolateTask", error: e, stackTrace: st); + } finally { + await closeHiveBoxes(); } - hiveHandler.setHistoryIds( - historyIds..removeWhere((id) => toRemoveIds.contains(id))); } } diff --git a/lib/services/hive_services.dart b/lib/services/hive_services.dart index 5ca476073..f098c1056 100644 --- a/lib/services/hive_services.dart +++ b/lib/services/hive_services.dart @@ -84,6 +84,25 @@ Future deleteHiveBoxes() async { } } +Future closeHiveBoxes() async { + try { + if (Hive.isBoxOpen(kDataBox)) { + await Hive.box(kDataBox).close(); + } + if (Hive.isBoxOpen(kEnvironmentBox)) { + await Hive.box(kEnvironmentBox).close(); + } + if (Hive.isBoxOpen(kHistoryMetaBox)) { + await Hive.box(kHistoryMetaBox).close(); + } + if (Hive.isBoxOpen(kHistoryLazyBox)) { + await Hive.lazyBox(kHistoryLazyBox).close(); + } + } catch (e, st) { + debugPrint("ERROR CLOSING HIVE BOXES: $e due to $st"); + } +} + final hiveHandler = HiveHandler(); class HiveHandler { diff --git a/lib/utils/history_utils.dart b/lib/utils/history_utils.dart index d1c3dbbc3..2e762ce42 100644 --- a/lib/utils/history_utils.dart +++ b/lib/utils/history_utils.dart @@ -1,9 +1,20 @@ import 'package:apidash/models/models.dart'; import 'package:apidash/consts.dart'; +import 'package:apidash/services/shared_preferences_services.dart' + show getSettingsFromSharedPrefs; +import 'package:path_provider/path_provider.dart' + show getApplicationDocumentsDirectory; import 'convert_utils.dart'; DateTime stripTime(DateTime dateTime) { - return DateTime(dateTime.year, dateTime.month, dateTime.day); + return DateTime( + dateTime.year, + dateTime.month, + dateTime.day, + dateTime.hour, + dateTime.minute, + dateTime.second, + ); } RequestModel getRequestModelFromHistoryModel(HistoryRequestModel model) { @@ -45,6 +56,16 @@ String? getLatestRequestId( return temporalGroups[keys.first]!.first.historyId; } +Future getHiveBoxPath() async { + if (!kIsDesktop) { + final dir = await getApplicationDocumentsDirectory(); + return dir.path; + } else { + final SettingsModel? settings = await getSettingsFromSharedPrefs(); + return settings?.workspaceFolderPath ?? ""; + } +} + DateTime getDateTimeKey(List keys, DateTime currentKey) { if (keys.isEmpty) return currentKey; for (DateTime key in keys) { @@ -115,11 +136,30 @@ List getRequestGroup( return requestGroup; } +int calculateOptimalBatchSize(int totalRecords) { + if (totalRecords < 100) return 10; + if (totalRecords < 500) return 50; + if (totalRecords < 5000) return 200; + return 500; +} + +List> createBatches(List items, int batchSize) { + return List.generate( + (items.length / batchSize).ceil(), + (index) => items.sublist( + index * batchSize, + (index * batchSize + batchSize).clamp(0, items.length), + ), + ); +} + DateTime? getRetentionDate(HistoryRetentionPeriod? retentionPeriod) { DateTime now = DateTime.now(); DateTime today = stripTime(now); switch (retentionPeriod) { + case HistoryRetentionPeriod.fiveSeconds: + return today.subtract(const Duration(seconds: 5)); case HistoryRetentionPeriod.oneWeek: return today.subtract(const Duration(days: 7)); case HistoryRetentionPeriod.oneMonth: From d954dc82196e40b51dbde019015314109d408201 Mon Sep 17 00:00:00 2001 From: Vishwa Karthik Date: Sun, 23 Feb 2025 19:20:01 +0530 Subject: [PATCH 2/3] revert the use of isolates / Compute functions due invalid study for #551 --- lib/consts.dart | 1 - lib/services/history_service.dart | 74 +++---------------------------- lib/utils/history_utils.dart | 21 ++------- 3 files changed, 9 insertions(+), 87 deletions(-) diff --git a/lib/consts.dart b/lib/consts.dart index 19be0a562..fbaa72da5 100644 --- a/lib/consts.dart +++ b/lib/consts.dart @@ -483,4 +483,3 @@ const kMsgClearHistory = const kMsgClearHistorySuccess = 'History cleared successfully'; const kMsgClearHistoryError = 'Error clearing history'; const kMsgShareError = "Unable to share"; -const int kIsolateThreshold = 50; diff --git a/lib/services/history_service.dart b/lib/services/history_service.dart index f1ad5a184..bdd21283c 100644 --- a/lib/services/history_service.dart +++ b/lib/services/history_service.dart @@ -1,13 +1,8 @@ import 'dart:developer'; -import 'dart:isolate'; import 'dart:async'; -import 'package:hive_flutter/hive_flutter.dart'; -import 'package:apidash/consts.dart'; -import 'package:flutter/foundation.dart' show compute; import 'package:apidash/models/models.dart'; import 'package:apidash/utils/utils.dart'; -import 'package:flutter/material.dart' show debugPrint; import 'hive_services.dart'; @@ -40,18 +35,10 @@ class HistoryServiceImpl implements HistoryService { if (toRemoveIds.isEmpty) return; int batchSize = calculateOptimalBatchSize(toRemoveIds.length); - final batches = createBatches(toRemoveIds, batchSize); - - final String hiveBoxPath = await getHiveBoxPath(); + final List> batches = createBatches(toRemoveIds, batchSize); for (var batch in batches) { - if (toRemoveIds.length < kIsolateThreshold) { - await compute(_deleteRecordsCompute, - {"batch": batch, "hiveBoxPath": hiveBoxPath}); - } else { - await _deleteRecordsWithIsolate( - batch: batch, hiveBoxPath: hiveBoxPath); - } + await deleteRecordsInBatches(batch); } hiveHandler.setHistoryIds(historyIds..removeWhere(toRemoveIds.contains)); @@ -61,66 +48,15 @@ class HistoryServiceImpl implements HistoryService { } } - Future _deleteRecordsCompute(Map args) async { - List batch = args['batch'] as List; - String? hiveBoxPath = args['hiveBoxPath'] as String; - - try { - Hive.init(hiveBoxPath); - await openHiveBoxes(); - - for (var id in batch) { - hiveHandler.deleteHistoryMeta(id); - hiveHandler.deleteHistoryRequest(id); - } - } catch (e, st) { - log("Unable to open hive box", - name: "_deleteRecordsCompute", error: e, stackTrace: st); - } finally { - await closeHiveBoxes(); - } - } - - Future _deleteRecordsWithIsolate({ - required List batch, - required String? hiveBoxPath, - }) async { - final receivePort = ReceivePort(); - - try { - await Isolate.spawn( - _isolateTask, [receivePort.sendPort, batch, hiveBoxPath]); - - await receivePort.first; - } catch (e, st) { - debugPrint("Unable to instantiate Isolate"); - log("Unable to instantiate Isolate", - name: "_deleteRecordsWithIsolate", error: e, stackTrace: st); - } finally { - receivePort.close(); - } - } - - Future _isolateTask(List args) async { - SendPort sendPort = args[0] as SendPort; - List batch = args[1] as List; - String? hiveBoxPath = args[2] as String; - - Hive.init(hiveBoxPath); - await openHiveBoxes(); - + static Future deleteRecordsInBatches(List batch) async { try { for (var id in batch) { hiveHandler.deleteHistoryMeta(id); hiveHandler.deleteHistoryRequest(id); } - sendPort.send(true); } catch (e, st) { - sendPort.send(false); - log("Isolate task failed", - name: "_isolateTask", error: e, stackTrace: st); - } finally { - await closeHiveBoxes(); + log("Error deleting records in batches", + name: "deleteRecordsInBatches", error: e, stackTrace: st); } } } diff --git a/lib/utils/history_utils.dart b/lib/utils/history_utils.dart index 2e762ce42..d2851c1f4 100644 --- a/lib/utils/history_utils.dart +++ b/lib/utils/history_utils.dart @@ -1,9 +1,6 @@ import 'package:apidash/models/models.dart'; import 'package:apidash/consts.dart'; -import 'package:apidash/services/shared_preferences_services.dart' - show getSettingsFromSharedPrefs; -import 'package:path_provider/path_provider.dart' - show getApplicationDocumentsDirectory; + import 'convert_utils.dart'; DateTime stripTime(DateTime dateTime) { @@ -56,16 +53,6 @@ String? getLatestRequestId( return temporalGroups[keys.first]!.first.historyId; } -Future getHiveBoxPath() async { - if (!kIsDesktop) { - final dir = await getApplicationDocumentsDirectory(); - return dir.path; - } else { - final SettingsModel? settings = await getSettingsFromSharedPrefs(); - return settings?.workspaceFolderPath ?? ""; - } -} - DateTime getDateTimeKey(List keys, DateTime currentKey) { if (keys.isEmpty) return currentKey; for (DateTime key in keys) { @@ -137,9 +124,9 @@ List getRequestGroup( } int calculateOptimalBatchSize(int totalRecords) { - if (totalRecords < 100) return 10; - if (totalRecords < 500) return 50; - if (totalRecords < 5000) return 200; + if (totalRecords < 100) return 50; + if (totalRecords < 500) return 80; + if (totalRecords < 5000) return 100; return 500; } From e110c79fed096b47bd6faa2936ff59a2972e885d Mon Sep 17 00:00:00 2001 From: Vishwa Karthik Date: Sun, 23 Feb 2025 19:24:28 +0530 Subject: [PATCH 3/3] removed unused code --- lib/services/hive_services.dart | 19 ------------------- 1 file changed, 19 deletions(-) diff --git a/lib/services/hive_services.dart b/lib/services/hive_services.dart index f098c1056..5ca476073 100644 --- a/lib/services/hive_services.dart +++ b/lib/services/hive_services.dart @@ -84,25 +84,6 @@ Future deleteHiveBoxes() async { } } -Future closeHiveBoxes() async { - try { - if (Hive.isBoxOpen(kDataBox)) { - await Hive.box(kDataBox).close(); - } - if (Hive.isBoxOpen(kEnvironmentBox)) { - await Hive.box(kEnvironmentBox).close(); - } - if (Hive.isBoxOpen(kHistoryMetaBox)) { - await Hive.box(kHistoryMetaBox).close(); - } - if (Hive.isBoxOpen(kHistoryLazyBox)) { - await Hive.lazyBox(kHistoryLazyBox).close(); - } - } catch (e, st) { - debugPrint("ERROR CLOSING HIVE BOXES: $e due to $st"); - } -} - final hiveHandler = HiveHandler(); class HiveHandler {