huge performance improvemnets and code optimizations & progressbar added

Author: rifsxd

pyproject.toml

@@ -2,7 +2,7 @@
 name = "pydvpl"
 description = "A CLI Tool Coded In Python3 To Convert WoTB ( Dava ) SmartDLC DVPL File Based On LZ4 High Compression."
 readme = "README.md"
-version = "0.1.0"
+version = "0.2.0"
 authors = [{ name = "RifsxD", email = "support@rxd-mods.xyz" }]
 license = { text = "MIT License" }
 requires-python = ">=3.10"

src/pydvpl/__init__.py

@@ -9,4 +9,4 @@
 
 __all__ = ['CompressDVPL', 'DecompressDVPL','ConvertDVPLFiles', 'VerifyDVPLFiles', 'readDVPLFooter', 'createDVPLFooter']
 
-__version__ = '0.1.0'
+__version__ = '0.2.0'

src/pydvpl/converter.py

@@ -1,11 +1,14 @@
-import os
 import argparse
 import time
 import os
 import lz4.block
 import zlib
 import threading
-from concurrent.futures import ThreadPoolExecutor
+import sys
+import multiprocessing
+import queue
+from functools import partial
+
 
 class Color:
     RED = '\033[31m'
@@ -14,28 +17,35 @@ class Color:
     YELLOW = '\033[33m'
     RESET = '\033[0m'
 
+
 class Meta:
     NAME = 'PyDVPL'
-    VERSION = '0.1.0'
-    DATE = '13/03/2024'
+    VERSION = '0.2.0'
+    DATE = '14/03/2024'
     DEV = 'RifsxD'
     REPO = 'https://github.com/rifsxd/pydvpl'
     INFO = 'A CLI Tool Coded In Python3 To Convert WoTB ( Dava ) SmartDLC DVPL File Based On LZ4 High Compression.'
 
+
 output_lock = threading.Lock()
 
+# Define a thread-safe queue to store processed files
+processed_queue = queue.Queue()
+
 DVPL_FOOTER_SIZE = 20
 DVPL_TYPE_NONE = 0
 DVPL_TYPE_LZ4 = 2
 DVPL_FOOTER = b"DVPL"
 
+
 class DVPLFooter:
     def __init__(self, original_size, compressed_size, crc32, type_val):
         self.original_size = original_size
         self.compressed_size = compressed_size
         self.crc32 = crc32
         self.type = type_val
 
+
 def createDVPLFooter(input_size, compressed_size, crc32_val, type_val):
     result = bytearray(DVPL_FOOTER_SIZE)
     result[:4] = input_size.to_bytes(4, 'little')
@@ -45,6 +55,7 @@ def createDVPLFooter(input_size, compressed_size, crc32_val, type_val):
     result[16:] = DVPL_FOOTER
     return result
 
+
 def readDVPLFooter(buffer):
     if len(buffer) < DVPL_FOOTER_SIZE:
         raise ValueError(Color.RED + "InvalidDVPLFooter: Buffer size is smaller than expected" + Color.RESET)
@@ -61,11 +72,13 @@ def readDVPLFooter(buffer):
 
     return DVPLFooter(original_size, compressed_size, crc32_val, type_val)
 
+
 def CompressDVPL(buffer):
     compressed_block = lz4.block.compress(buffer, store_size=False)
     footer_buffer = createDVPLFooter(len(buffer), len(compressed_block), zlib.crc32(compressed_block), DVPL_TYPE_LZ4)
     return compressed_block + footer_buffer
 
+
 def DecompressDVPL(buffer):
     footer_data = readDVPLFooter(buffer)
     target_block = buffer[:-DVPL_FOOTER_SIZE]
@@ -88,22 +101,39 @@ def DecompressDVPL(buffer):
     else:
         raise ValueError(Color.RED + "UNKNOWN DVPL FORMAT" + Color.RESET)
 
-def ConvertDVPLFiles(directory_or_file, config):
+
+def print_progress_bar(processed_files, total_files):
+    with output_lock:
+        progress = min(processed_files.value / total_files, 1.0)  # Ensure progress doesn't exceed 100%
+        bar_length = 50
+        filled_length = int(bar_length * progress)
+        bar = '=' * filled_length + '-' * (bar_length - filled_length)
+        percentage = progress * 100
+        sys.stdout.write('\rProcessing: [{:<50}] {:.2f}%'.format(bar, percentage))
+        sys.stdout.flush()
+
+
+def count_total_files(directory):
+    total_files = 0
+    for root, dirs, files in os.walk(directory):
+        total_files += len(files)
+    return total_files
+
+
+def ConvertDVPLFiles(directory_or_file, config, total_files, processed_files):
     success_count = 0
     failure_count = 0
     ignored_count = 0
 
     if os.path.isdir(directory_or_file):
         dir_list = os.listdir(directory_or_file)
-        with ThreadPoolExecutor() as executor:
-            future_results = []
-            for dir_item in dir_list:
-                future_results.append(executor.submit(ConvertDVPLFiles, os.path.join(directory_or_file, dir_item), config))
-            for future in future_results:
-                succ, fail, ignored = future.result()
-                success_count += succ
-                failure_count += fail
-                ignored_count += ignored
+        for dir_item in dir_list:
+            succ, fail, ignored = ConvertDVPLFiles(os.path.join(directory_or_file, dir_item), config, total_files, processed_files)
+            success_count += succ
+            failure_count += fail
+            ignored_count += ignored
+            processed_files.value += 1
+            print_progress_bar(processed_files, total_files)
     else:
         is_decompression = config.mode == "decompress" and directory_or_file.endswith(".dvpl")
         is_compression = config.mode == "compress" and not directory_or_file.endswith(".dvpl")
@@ -147,22 +177,24 @@ def ConvertDVPLFiles(directory_or_file, config):
 
     return success_count, failure_count, ignored_count
 
-def VerifyDVPLFiles(directory_or_file, config):
+
+
+def VerifyDVPLFiles(directory_or_file, config, total_files, processed_files):
     success_count = 0
     failure_count = 0
     ignored_count = 0
 
     if os.path.isdir(directory_or_file):
         dir_list = os.listdir(directory_or_file)
-        with ThreadPoolExecutor() as executor:
-            future_results = []
-            for dir_item in dir_list:
-                future_results.append(executor.submit(VerifyDVPLFiles, os.path.join(directory_or_file, dir_item), config))
-            for future in future_results:
-                succ, fail, ignored = future.result()
-                success_count += succ
-                failure_count += fail
-                ignored_count += ignored
+        for dir_item in dir_list:
+            succ, fail, ignored = VerifyDVPLFiles(os.path.join(directory_or_file, dir_item), config, total_files,
+                                                  processed_files)
+            success_count += succ
+            failure_count += fail
+            ignored_count += ignored
+            if processed_files.value < total_files:  # Ensure processed files count does not exceed total files
+                processed_files.value += 1
+            print_progress_bar(processed_files, total_files)
     else:
         is_dvpl_file = directory_or_file.endswith(".dvpl")
 
@@ -187,7 +219,8 @@ def VerifyDVPLFiles(directory_or_file, config):
 
                 if config.verbose:
                     with output_lock:
-                        print(f"{Color.GREEN}\nFile{Color.RESET} {file_path} has been successfully {Color.GREEN}verified.{Color.RESET}")
+                        print(
+                            f"{Color.GREEN}\nFile{Color.RESET} {file_path} has been successfully {Color.GREEN}verified.{Color.RESET}")
 
                 success_count += 1
             except Exception as e:
@@ -203,13 +236,18 @@ def VerifyDVPLFiles(directory_or_file, config):
 
     return success_count, failure_count, ignored_count
 
+
 def ParseCommandLineArgs():
     parser = argparse.ArgumentParser()
-    parser.add_argument("-m", "--mode", help="mode can be 'compress' / 'decompress' / 'help' (for an extended help guide).")
-    parser.add_argument("-k", "--keep-originals", action="store_true", help="keep original files after compression/decompression.")
-    parser.add_argument("-v", "--verbose", action="store_true", help="shows verbose information for all processed files.")
+    parser.add_argument("-m", "--mode",
+                        help="mode can be 'compress' / 'decompress' / 'help' (for an extended help guide).")
+    parser.add_argument("-k", "--keep-originals", action="store_true",
+                        help="keep original files after compression/decompression.")
+    parser.add_argument("-v", "--verbose", action="store_true",
+                        help="shows verbose information for all processed files.")
     parser.add_argument("-p", "--path", help="directory/files path to process. Default is the current directory.")
-    parser.add_argument("-i", "--ignore", default="", help="Comma-separated list of file extensions to ignore during compression.")
+    parser.add_argument("-i", "--ignore", default="",
+                        help="Comma-separated list of file extensions to ignore during compression.")
     args = parser.parse_args()
 
     if not args.mode:
@@ -220,6 +258,7 @@ def ParseCommandLineArgs():
 
     return args
 
+
 def PrintHelpMessage():
     print('''pydvpl [--mode] [--keep-originals] [--path]
 
@@ -263,12 +302,13 @@ def PrintHelpMessage():
         $ pydvpl --mode compress --path /path/to/decompress --ignore exe,dll
 
         $ pydvpl --mode compress --path /path/to/decompress --ignore test.exe,test.txt
-        
+
         $ pydvpl --mode verify -path /path/to/verify
 
         $ pydvpl --mode verify -path /path/to/verify/verify.yaml.dvpl
     ''')
 
+
 def PrintElapsedTime(elapsed_time):
     if elapsed_time < 1:
         print(f"\nProcessing took {Color.GREEN}{int(elapsed_time * 1000)} ms{Color.RESET}\n")
@@ -277,6 +317,7 @@ def PrintElapsedTime(elapsed_time):
     else:
         print(f"\nProcessing took {Color.RED}{int(elapsed_time / 60)} min{Color.RESET}\n")
 
+
 def main():
     print(f"\n{Color.BLUE}• Name:{Color.RESET} {Meta.NAME}")
     print(f"{Color.BLUE}• Version:{Color.RESET} {Meta.VERSION}")
@@ -288,22 +329,41 @@ def main():
     start_time = time.time()
     config = ParseCommandLineArgs()
 
+    total_files = count_total_files(config.path)
+    manager = multiprocessing.Manager()
+    processed_files = manager.Value('i', 0)  # Define processed_files using a Manager
+
     try:
         if config.mode in ["compress", "decompress"]:
-            success_count, failure_count, ignored_count = ConvertDVPLFiles(config.path, config)
-            print(f"\n{Color.GREEN}{config.mode.upper()} FINISHED{Color.RESET}. Successful conversions: {Color.GREEN}{success_count}{Color.RESET}, Failed conversions: {Color.RED}{failure_count}{Color.RESET}, Ignored files: {Color.YELLOW}{ignored_count}{Color.RESET}")
+            process_func = partial(ConvertDVPLFiles, config=config, total_files=total_files,
+                                   processed_files=processed_files)
         elif config.mode == "verify":
-            success_count, failure_count, ignored_count = VerifyDVPLFiles(config.path, config)
-            print(f"\n{Color.GREEN}{config.mode.upper()} FINISHED{Color.RESET}. Successful verifications: {Color.GREEN}{success_count}{Color.RESET}, Failed verifications: {Color.RED}{failure_count}{Color.RESET}, Ignored files: {Color.YELLOW}{ignored_count}{Color.RESET}")
-        elif config.mode == "help":
-            PrintHelpMessage()
+            process_func = partial(VerifyDVPLFiles, config=config, total_files=total_files,
+                                   processed_files=processed_files)
         else:
             raise ValueError("Incorrect mode selected. Use '--help' for information.")
+
+        with multiprocessing.Pool() as pool:
+            results = pool.map(process_func, [config.path])
+
+        success_count = sum(result[0] for result in results)
+        failure_count = sum(result[1] for result in results)
+        ignored_count = sum(result[2] for result in results)
+
+        if config.mode in ["compress", "decompress"]:
+            print(
+                f"\n\n{Color.GREEN}{config.mode.upper()} FINISHED{Color.RESET}. Successful conversions: {Color.GREEN}{success_count}{Color.RESET}, Failed conversions: {Color.RED}{failure_count}{Color.RESET}, Ignored files: {Color.YELLOW}{ignored_count}{Color.RESET}")
+        elif config.mode == "verify":
+            print(
+                f"\n\n{Color.GREEN}{config.mode.upper()} FINISHED{Color.RESET}. Successful verifications: {Color.GREEN}{success_count}{Color.RESET}, Failed verifications: {Color.RED}{failure_count}{Color.RESET}, Ignored files: {Color.YELLOW}{ignored_count}{Color.RESET}")
+        elif config.mode == "help":
+            PrintHelpMessage()
     except Exception as e:
-        print(f"\n{Color.RED}{config.mode.upper()} FAILED{Color.RESET}: {e}\n")
+        print(f"\n\n{Color.RED}{config.mode.upper()} FAILED{Color.RESET}: {e}\n")
 
     elapsed_time = time.time() - start_time
     PrintElapsedTime(elapsed_time)
 
+
 if __name__ == "__main__":
     main()