Add Image Deduplicator option for #2

joyrider3774 · Sep 4, 2021 · 071eceb · 071eceb
1 parent fc51487
commit 071eceb
Show file tree

Hide file tree

Showing 46 changed files with 135 additions and 26 deletions.
diff --git a/FileDeDuplicator.cs b/FileDeDuplicator.cs
@@ -0,0 +1,51 @@
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Security.Cryptography;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace HtmlExporterPlugin
+{
+    class FileDeDuplicator
+    {
+        private Dictionary<string, string> DuplicateDictionary = new Dictionary<string, string>();
+
+        private static string CalculateMD5(string filename)
+        {
+            using (var md5 = MD5.Create())
+            {
+                using (var stream = new BufferedStream(File.OpenRead(filename), 1048576))
+                {
+                    var hash = md5.ComputeHash(stream);
+                    return BitConverter.ToString(hash).Replace("-", "").ToLowerInvariant();
+                }
+            }
+        }
+
+        public void clear()
+        {
+            DuplicateDictionary.Clear();
+        }
+
+        public string GetUniqueFile(string fullfilename, string storedfilename, bool DoCheck)
+        {
+            if (!DoCheck || !File.Exists(fullfilename))
+            {
+                return storedfilename;
+            }
+
+            string md5 = CalculateMD5(fullfilename);
+            if (DuplicateDictionary.ContainsKey(md5))
+            {
+                return DuplicateDictionary[md5];
+            }
+            else
+            {
+                DuplicateDictionary[md5] = storedfilename;
+                return storedfilename;
+            }
+        }
+    }
+}
diff --git a/HtmlExporterPlugin.cs b/HtmlExporterPlugin.cs
@@ -13,6 +13,7 @@
 using System.Threading;
 using System.Web;
 using System.Windows.Controls;
+using System.Security.Cryptography;
 //using ZetaProducerHtmlCompressor.Internal;
 
 namespace HtmlExporterPlugin
@@ -121,6 +122,7 @@ public void DoExportToHtml()
                       Dictionary<string, string> GameMediaHashtable = new Dictionary<string, string>();
                       Dictionary<string, bool> GameMediaCopyDoneDict = new Dictionary<string, bool>();
                       Dictionary<string, string> FirstGroupFieldFileNames = new Dictionary<string, string>();
+                      FileDeDuplicator DeDuplicator = new FileDeDuplicator();
                       int pagecount = Settings.Pages.Count;
                       int PageNr = 0;
                       int Errors = 0;
@@ -1134,6 +1136,7 @@ public void DoExportToHtml()
                                                   gameicon = realgame.Platform.Icon.Replace("\\", "/");
                                               }
                                           }
+                                          gameicon = DeDuplicator.GetUniqueFile(PlayniteApi.Database.GetFullFilePath(gameicon), gameicon, Settings.ConvertImageOptions.DetectDuplicates);
                                       }
                                       GameMediaHashtable[Constants.MediaIconText + realgame.Id.ToString()] = gameicon;
                                   }
@@ -1169,6 +1172,8 @@ public void DoExportToHtml()
                                                   coverimage = realgame.Platform.Cover.Replace("\\", "/");
                                               }
                                           }
+
+                                          coverimage = DeDuplicator.GetUniqueFile(PlayniteApi.Database.GetFullFilePath(coverimage), coverimage, Settings.ConvertImageOptions.DetectDuplicates);
                                       }
                                       GameMediaHashtable[Constants.MediaCoverText + realgame.Id.ToString()] = coverimage;
                                   }
@@ -1190,17 +1195,20 @@ public void DoExportToHtml()
                                       {
                                           backgroundimage = filespathbackground;
                                       }
-                                      else
-                                          if ((filespathbackground != String.Empty) && File.Exists(PlayniteApi.Database.GetFullFilePath(filespathbackground)))
-                                      {
-                                          backgroundimage = realgame.BackgroundImage.Replace("\\", "/");
-                                      }
                                       else
                                       {
-                                          if (!String.IsNullOrEmpty(realgame.Platform?.Background))
+                                          if ((filespathbackground != String.Empty) && File.Exists(PlayniteApi.Database.GetFullFilePath(filespathbackground)))
                                           {
-                                              backgroundimage = realgame.Platform.Background.Replace("\\", "/");
+                                              backgroundimage = realgame.BackgroundImage.Replace("\\", "/");
+                                          }
+                                          else
+                                          {
+                                              if (!String.IsNullOrEmpty(realgame.Platform?.Background))
+                                              {
+                                                  backgroundimage = realgame.Platform.Background.Replace("\\", "/");
+                                              }
                                           }
+                                          backgroundimage = DeDuplicator.GetUniqueFile(PlayniteApi.Database.GetFullFilePath(backgroundimage), backgroundimage, Settings.ConvertImageOptions.DetectDuplicates);
                                       }
                                       GameMediaHashtable[Constants.MediaBackgroundText + realgame.Id.ToString()] = backgroundimage;
                                   }

diff --git a/HtmlExporterPlugin.csproj b/HtmlExporterPlugin.csproj
@@ -55,6 +55,7 @@
     <Reference Include="WindowsBase" />
   </ItemGroup>
   <ItemGroup>
+    <Compile Include="FileDeDuplicator.cs" />
     <Compile Include="OriginalImageDataFile.cs" />
     <Compile Include="OriginalGameImageData.cs" />
     <Compile Include="ImageProcessRunner.cs" />

diff --git a/HtmlExporterPluginSettingsView.xaml.cs b/HtmlExporterPluginSettingsView.xaml.cs
@@ -261,7 +261,7 @@ private void ButImageOptions_Click(object sender, RoutedEventArgs e)
             });
 
 
-            window.Height = 450;
+            window.Height = 475;
             window.Width = 800;
             window.Title = Constants.ImageOptionsText;
 

diff --git a/ImageOptions.cs b/ImageOptions.cs
@@ -32,14 +32,15 @@ public class ImageOptions
 
         public bool AlwaysProcess { get; set; } = false;
 
+        public bool DetectDuplicates { get; set; } = false;
 
         public string GetUniqueString(bool includeMaxTasks = false, bool includeAlwaysProcess = false)
         {
             return ConvertToJpg.ToString() + '_' + ConvertToPng.ToString() + '_' + JpgQuality.ToString() + '_' +
                 ResizeCoverImage.ToString() + '_' + CoverImageWidth + '_' + CoverImageHeight + '_' +
                 ResizeBackgroundImage.ToString() + '_' + BackgroundImageWidth + '_' + BackgroundImageHeight + '_' +
                 ResizeIconImage.ToString() + '_' + IconImageWidth + '_' + IconImageHeight + '_' +
-                ForceConversion.ToString() + (includeMaxTasks ? '_' + MaxTasks.ToString() : String.Empty) +
+                ForceConversion.ToString() + '_' + DetectDuplicates.ToString() + (includeMaxTasks ? '_' + MaxTasks.ToString() : String.Empty) +
                 (includeAlwaysProcess ? '_' + AlwaysProcess.ToString() : String.Empty);
         }
 

diff --git a/ImageOptionsView.xaml b/ImageOptionsView.xaml
@@ -24,6 +24,9 @@
                 <StackPanel Orientation="Horizontal"  Margin="0,0,5,5">
                     <CheckBox x:Name="ChkAwalysProcess" IsChecked="{Binding AlwaysProcess}" Content="{DynamicResource LOC_HTMLEXPORTER_AlwaysProcess}" VerticalAlignment="Center" Margin="0,0,5,0"/>
                 </StackPanel>
+                <StackPanel Orientation="Horizontal"  Margin="0,0,5,5">
+                    <CheckBox x:Name="ChkDetectDuplicates" IsChecked="{Binding DetectDuplicates}" Content="{DynamicResource LOC_HTMLEXPORTER_DetectDuplicates}" VerticalAlignment="Center" Margin="0,0,5,0"/>
+                </StackPanel>
                 <StackPanel Orientation="Horizontal"  Margin="0,0,5,5">
                     <CheckBox x:Name="ChkResizeCoverImage" IsChecked="{Binding ResizeCoverImage}" Content="{DynamicResource LOC_HTMLEXPORTER_ResizeCoverImage}" VerticalAlignment="Center" Margin="0,0,5,0"/>
                     <TextBox Height="23" PreviewTextInput="NumberValidationTextBox" Text="{Binding CoverImageWidth}" x:Name="tbCoverImageWidth" HorizontalContentAlignment="Stretch" Width="50">

diff --git a/ImageProcessRunner.cs b/ImageProcessRunner.cs
@@ -45,7 +45,7 @@ public ImageProcessDef(ImageProcessDef Source)
         public int FailedConverts = 0;
         public List<string> FailedFiles = new List<string>();
 
-        private readonly List<ImageProcessDef> ImagesProcessDefList = new List<ImageProcessDef>();
+        private readonly Dictionary<string, ImageProcessDef> ImagesProcessDefDict = new Dictionary<string, ImageProcessDef>();
         private CancellationTokenSource cancellationTokenSource = null;  
 
         private static int StartProcessWait(string path, string arguments, string workDir, bool noWindow = false)
@@ -77,22 +77,25 @@ private static int StartProcessWait(string path, string arguments, string workDi
 
         public void addImageProcess(bool copyonly, string source, string dest, string destconverted, string exepath, string arguments, string workDir, bool noWindow = false)
         {
-            ImageProcessDef ImageProcess = new ImageProcessDef();
-            ImageProcess.path = exepath;
-            ImageProcess.arguments = arguments;
-            ImageProcess.workDir = workDir;
-            ImageProcess.noWindow = noWindow;
-            ImageProcess.copyonly = copyonly;
-            ImageProcess.source = source;
-            ImageProcess.dest = dest;
-            ImageProcess.destconverted = destconverted;
-            ImagesProcessDefList.Add(ImageProcess);
+            if (!ImagesProcessDefDict.ContainsKey(source))
+            {
+                ImageProcessDef ImageProcess = new ImageProcessDef();
+                ImageProcess.path = exepath;
+                ImageProcess.arguments = arguments;
+                ImageProcess.workDir = workDir;
+                ImageProcess.noWindow = noWindow;
+                ImageProcess.copyonly = copyonly;
+                ImageProcess.source = source;
+                ImageProcess.dest = dest;
+                ImageProcess.destconverted = destconverted;
+                ImagesProcessDefDict.Add(source, ImageProcess);
+            }
 
         }
 
         public void Clear()
         {
-            ImagesProcessDefList.Clear();
+            ImagesProcessDefDict.Clear();
         }
 
         public void Stop()
@@ -105,14 +108,14 @@ public void Stop()
 
         public int Count()
         {
-            return ImagesProcessDefList.Count;
+            return ImagesProcessDefDict.Count;
         }
 
         public void Start(int maxConcurrency, DelCallback CallBack)
         {
             using (SemaphoreSlim concurrencySemaphore = new SemaphoreSlim(maxConcurrency))
             {
-                if (ImagesProcessDefList.Count == 0)
+                if (ImagesProcessDefDict.Count == 0)
                 {
                     return;
                 }
@@ -121,9 +124,9 @@ public void Start(int maxConcurrency, DelCallback CallBack)
                 FailedConverts = 0;
                 SuccesFullConverts = 0;
                 FailedFiles.Clear();
-                int max = ImagesProcessDefList.Count();
+                int max = ImagesProcessDefDict.Count();
                 List<Task<bool>> tasks = new List<Task<bool>>();
-                foreach (ImageProcessDef ImageProcess in ImagesProcessDefList)
+                foreach (ImageProcessDef ImageProcess in ImagesProcessDefDict.Values)
                 {
                     concurrencySemaphore.Wait();
                     Task<bool> t = Task.Factory.StartNew((object o) =>
@@ -186,6 +189,10 @@ public void Start(int maxConcurrency, DelCallback CallBack)
                                 return File.Exists(ImageProcessDefTask.destconverted);
                             }
                         }
+                        catch
+                        {
+                            return false;
+                        }
                         finally
                         {
                             _ = concurrencySemaphore.Release();

diff --git a/Localization/LocSource.xaml b/Localization/LocSource.xaml
@@ -189,6 +189,7 @@
     <sys:String x:Key="LOC_HTMLEXPORTER_TotalImagesProcess">Total images to process:</sys:String>
     <sys:String x:Key="LOC_HTMLEXPORTER_AlwaysProcess">Always process images, even if they have not changed</sys:String>
     <sys:String x:Key="LOC_HTMLEXPORTER_Optimizations">optimizations</sys:String>
+    <sys:String x:Key="LOC_HTMLEXPORTER_DetectDuplicates">Detect Duplicate Images</sys:String>
 
 
 

diff --git a/Localization/af_ZA.xaml b/Localization/af_ZA.xaml
@@ -189,6 +189,7 @@
     <sys:String x:Key="LOC_HTMLEXPORTER_TotalImagesProcess">Total images to process:</sys:String>
     <sys:String x:Key="LOC_HTMLEXPORTER_AlwaysProcess">Always process images, even if they have not changed</sys:String>
     <sys:String x:Key="LOC_HTMLEXPORTER_Optimizations">optimizations</sys:String>
+    <sys:String x:Key="LOC_HTMLEXPORTER_DetectDuplicates">Detect Duplicate Images</sys:String>
 
 
 

diff --git a/Localization/ar_SA.xaml b/Localization/ar_SA.xaml
@@ -189,6 +189,7 @@
     <sys:String x:Key="LOC_HTMLEXPORTER_TotalImagesProcess">Total images to process:</sys:String>
     <sys:String x:Key="LOC_HTMLEXPORTER_AlwaysProcess">Always process images, even if they have not changed</sys:String>
     <sys:String x:Key="LOC_HTMLEXPORTER_Optimizations">optimizations</sys:String>
+    <sys:String x:Key="LOC_HTMLEXPORTER_DetectDuplicates">Detect Duplicate Images</sys:String>
 
 
 

diff --git a/Localization/ca_ES.xaml b/Localization/ca_ES.xaml
@@ -189,6 +189,7 @@
     <sys:String x:Key="LOC_HTMLEXPORTER_TotalImagesProcess">Total images to process:</sys:String>
     <sys:String x:Key="LOC_HTMLEXPORTER_AlwaysProcess">Always process images, even if they have not changed</sys:String>
     <sys:String x:Key="LOC_HTMLEXPORTER_Optimizations">optimizations</sys:String>
+    <sys:String x:Key="LOC_HTMLEXPORTER_DetectDuplicates">Detect Duplicate Images</sys:String>
 
 
 

diff --git a/Localization/cs_CZ.xaml b/Localization/cs_CZ.xaml
@@ -189,6 +189,7 @@
     <sys:String x:Key="LOC_HTMLEXPORTER_TotalImagesProcess">Total images to process:</sys:String>
     <sys:String x:Key="LOC_HTMLEXPORTER_AlwaysProcess">Always process images, even if they have not changed</sys:String>
     <sys:String x:Key="LOC_HTMLEXPORTER_Optimizations">optimizations</sys:String>
+    <sys:String x:Key="LOC_HTMLEXPORTER_DetectDuplicates">Detect Duplicate Images</sys:String>
 
 
 

diff --git a/Localization/da_DK.xaml b/Localization/da_DK.xaml
@@ -189,6 +189,7 @@
     <sys:String x:Key="LOC_HTMLEXPORTER_TotalImagesProcess">Total images to process:</sys:String>
     <sys:String x:Key="LOC_HTMLEXPORTER_AlwaysProcess">Always process images, even if they have not changed</sys:String>
     <sys:String x:Key="LOC_HTMLEXPORTER_Optimizations">optimizations</sys:String>
+    <sys:String x:Key="LOC_HTMLEXPORTER_DetectDuplicates">Detect Duplicate Images</sys:String>
 
 
 

diff --git a/Localization/de_DE.xaml b/Localization/de_DE.xaml
@@ -189,6 +189,7 @@
     <sys:String x:Key="LOC_HTMLEXPORTER_TotalImagesProcess">Gesamte zu verarbeitende Bilder:</sys:String>
     <sys:String x:Key="LOC_HTMLEXPORTER_AlwaysProcess">Bilder immer verarbeiten, auch wenn sie nicht geändert wurden</sys:String>
     <sys:String x:Key="LOC_HTMLEXPORTER_Optimizations">optimierungen</sys:String>
+    <sys:String x:Key="LOC_HTMLEXPORTER_DetectDuplicates">Doppelte Bilder erkennen</sys:String>
 
 
 

diff --git a/Localization/el_GR.xaml b/Localization/el_GR.xaml
@@ -189,6 +189,7 @@
     <sys:String x:Key="LOC_HTMLEXPORTER_TotalImagesProcess">Total images to process:</sys:String>
     <sys:String x:Key="LOC_HTMLEXPORTER_AlwaysProcess">Always process images, even if they have not changed</sys:String>
     <sys:String x:Key="LOC_HTMLEXPORTER_Optimizations">optimizations</sys:String>
+    <sys:String x:Key="LOC_HTMLEXPORTER_DetectDuplicates">Detect Duplicate Images</sys:String>
 
 
 

diff --git a/Localization/en_US.xaml b/Localization/en_US.xaml
@@ -189,6 +189,7 @@
     <sys:String x:Key="LOC_HTMLEXPORTER_TotalImagesProcess">Total images to process:</sys:String>
     <sys:String x:Key="LOC_HTMLEXPORTER_AlwaysProcess">Always process images, even if they have not changed</sys:String>
     <sys:String x:Key="LOC_HTMLEXPORTER_Optimizations">optimizations</sys:String>
+    <sys:String x:Key="LOC_HTMLEXPORTER_DetectDuplicates">Detect Duplicate Images</sys:String>
 
 
 

diff --git a/Localization/eo_UY.xaml b/Localization/eo_UY.xaml
@@ -189,6 +189,7 @@
     <sys:String x:Key="LOC_HTMLEXPORTER_TotalImagesProcess">Total images to process:</sys:String>
     <sys:String x:Key="LOC_HTMLEXPORTER_AlwaysProcess">Always process images, even if they have not changed</sys:String>
     <sys:String x:Key="LOC_HTMLEXPORTER_Optimizations">optimizations</sys:String>
+    <sys:String x:Key="LOC_HTMLEXPORTER_DetectDuplicates">Detect Duplicate Images</sys:String>
 
 
 

diff --git a/Localization/es_ES.xaml b/Localization/es_ES.xaml
@@ -189,6 +189,7 @@
     <sys:String x:Key="LOC_HTMLEXPORTER_TotalImagesProcess">Total de imágenes para procesar:</sys:String>
     <sys:String x:Key="LOC_HTMLEXPORTER_AlwaysProcess">Siempre procesar imágenes, incluso si no han cambiado</sys:String>
     <sys:String x:Key="LOC_HTMLEXPORTER_Optimizations">optimizaciones</sys:String>
+    <sys:String x:Key="LOC_HTMLEXPORTER_DetectDuplicates">Detectar imágenes duplicadas</sys:String>
 
 
 

diff --git a/Localization/et_EE.xaml b/Localization/et_EE.xaml
@@ -189,6 +189,7 @@
     <sys:String x:Key="LOC_HTMLEXPORTER_TotalImagesProcess">Total images to process:</sys:String>
     <sys:String x:Key="LOC_HTMLEXPORTER_AlwaysProcess">Always process images, even if they have not changed</sys:String>
     <sys:String x:Key="LOC_HTMLEXPORTER_Optimizations">optimizations</sys:String>
+    <sys:String x:Key="LOC_HTMLEXPORTER_DetectDuplicates">Detect Duplicate Images</sys:String>
 
 
 

diff --git a/Localization/fa_IR.xaml b/Localization/fa_IR.xaml
@@ -189,6 +189,7 @@
     <sys:String x:Key="LOC_HTMLEXPORTER_TotalImagesProcess">Total images to process:</sys:String>
     <sys:String x:Key="LOC_HTMLEXPORTER_AlwaysProcess">Always process images, even if they have not changed</sys:String>
     <sys:String x:Key="LOC_HTMLEXPORTER_Optimizations">optimizations</sys:String>
+    <sys:String x:Key="LOC_HTMLEXPORTER_DetectDuplicates">Detect Duplicate Images</sys:String>
 
 
 

diff --git a/Localization/fi_FI.xaml b/Localization/fi_FI.xaml
@@ -189,6 +189,7 @@
     <sys:String x:Key="LOC_HTMLEXPORTER_TotalImagesProcess">Total images to process:</sys:String>
     <sys:String x:Key="LOC_HTMLEXPORTER_AlwaysProcess">Always process images, even if they have not changed</sys:String>
     <sys:String x:Key="LOC_HTMLEXPORTER_Optimizations">optimizations</sys:String>
+    <sys:String x:Key="LOC_HTMLEXPORTER_DetectDuplicates">Detect Duplicate Images</sys:String>
 
 
 

diff --git a/Localization/fr_FR.xaml b/Localization/fr_FR.xaml
@@ -189,6 +189,7 @@
     <sys:String x:Key="LOC_HTMLEXPORTER_TotalImagesProcess">Total des images à traiter:</sys:String>
     <sys:String x:Key="LOC_HTMLEXPORTER_AlwaysProcess">Toujours traiter les images, même si elles n'ont pas changé</sys:String>
     <sys:String x:Key="LOC_HTMLEXPORTER_Optimizations">optimizations</sys:String>
+    <sys:String x:Key="LOC_HTMLEXPORTER_DetectDuplicates">Détecter les doublons d'images</sys:String>
 
 
 

diff --git a/Localization/he_IL.xaml b/Localization/he_IL.xaml
@@ -189,6 +189,7 @@
     <sys:String x:Key="LOC_HTMLEXPORTER_TotalImagesProcess">Total images to process:</sys:String>
     <sys:String x:Key="LOC_HTMLEXPORTER_AlwaysProcess">Always process images, even if they have not changed</sys:String>
     <sys:String x:Key="LOC_HTMLEXPORTER_Optimizations">optimizations</sys:String>
+    <sys:String x:Key="LOC_HTMLEXPORTER_DetectDuplicates">Detect Duplicate Images</sys:String>
 
 
 

diff --git a/Localization/hr_HR.xaml b/Localization/hr_HR.xaml
@@ -189,6 +189,7 @@
     <sys:String x:Key="LOC_HTMLEXPORTER_TotalImagesProcess">Total images to process:</sys:String>
     <sys:String x:Key="LOC_HTMLEXPORTER_AlwaysProcess">Always process images, even if they have not changed</sys:String>
     <sys:String x:Key="LOC_HTMLEXPORTER_Optimizations">optimizations</sys:String>
+    <sys:String x:Key="LOC_HTMLEXPORTER_DetectDuplicates">Detect Duplicate Images</sys:String>
 
 
 

diff --git a/Localization/hu_HU.xaml b/Localization/hu_HU.xaml
@@ -189,6 +189,7 @@
     <sys:String x:Key="LOC_HTMLEXPORTER_TotalImagesProcess">Total images to process:</sys:String>
     <sys:String x:Key="LOC_HTMLEXPORTER_AlwaysProcess">Always process images, even if they have not changed</sys:String>
     <sys:String x:Key="LOC_HTMLEXPORTER_Optimizations">optimizations</sys:String>
+    <sys:String x:Key="LOC_HTMLEXPORTER_DetectDuplicates">Detect Duplicate Images</sys:String>