3.0.0.1

Added DownloadLimitDate, update and Reset functions in the plugin, DateLimit property in the IPluginContentProvider
Fixed donation button link, images loading error, cookies error, instagram settings form, minor bugs.

Author: AAndyProgram

Changelog.md

@@ -1,3 +1,16 @@
+# 3.0.0.1
+
+- Added
+  - Download data up to a specific date
+  - Update and Reset functions in the plugin (ISiteSettings)
+  - PostsDateLimit propperty in the plugin (IPluginContentProvider)
+- Fixed
+  - The donation button redirects to a broken link
+  - In some cases, an error occurs when fast loading images
+  - In some cases, cookies are not saved
+  - Some design fixes
+  - Minor bugs
+
 # 3.0.0.0
 
 **Attention! This version of the program makes changes user data file (Users.xml). Once you start using this version, you will not be able to use previous versions of the program. Therefore, it is highly recommended to archive the program settings folder and archive the users' data files (you can use the [```ArchiveSCrawlerUsersDataFiles.bat```](Tools/ArchiveSCrawlerUsersDataFiles.bat) tool to archive the data files of all users).**

ProgramScreenshots/MainContext2.png

@@ -26,6 +26,7 @@ Namespace Plugin
         Property SeparateVideoFolder As Boolean
         Property DataPath As String
         Property PostsNumberLimit As Integer?
+        Property PostsDateLimit As Date?
         Function ExchangeOptionsGet() As Object
         Sub ExchangeOptionsSet(ByVal Obj As Object)
         Sub XmlFieldsSet(ByVal Fields As List(Of KeyValuePair(Of String, String)))

SCrawler.PluginProvider/Interfaces/IPluginContentProvider.vb

@@ -41,6 +41,8 @@ Namespace Plugin
         Sub AfterDownload(ByVal User As Object, ByVal What As Download)
         Sub DownloadDone(ByVal What As Download)
 #End Region
+        Sub Update()
+        Sub Reset()
         Sub OpenSettingsForm()
         Sub UserOptions(ByRef Options As Object, ByVal OpenForm As Boolean)
     End Interface

SCrawler.PluginProvider/Interfaces/ISiteSettings.vb

@@ -32,6 +32,6 @@ Imports System.Runtime.InteropServices
 ' by using the '*' as shown below:
 ' <Assembly: AssemblyVersion("1.0.*")>
 
-<Assembly: AssemblyVersion("3.0.0.0")>
-<Assembly: AssemblyFileVersion("3.0.0.0")>
+<Assembly: AssemblyVersion("3.0.0.1")>
+<Assembly: AssemblyFileVersion("3.0.0.1")>
 <Assembly: NeutralResourcesLanguage("en")>

SCrawler.PluginProvider/My Project/AssemblyInfo.vb

@@ -97,6 +97,11 @@ Namespace API.Base
             Return True
         End Function
 #End Region
+        Friend Overridable Sub Update() Implements ISiteSettings.Update
+            If Not Responser Is Nothing Then Responser.SaveSettings()
+        End Sub
+        Friend Overridable Sub Reset() Implements ISiteSettings.Reset
+        End Sub
         Friend Overridable Sub UserOptions(ByRef Options As Object, ByVal OpenForm As Boolean) Implements ISiteSettings.UserOptions
             Options = Nothing
         End Sub

SCrawler/API/Base/SiteSettingsBase.vb

@@ -711,6 +711,7 @@ BlockNullPicture:
 #End Region
 #Region "Download functions and options"
         Friend Overridable Property DownloadTopCount As Integer? = Nothing Implements IUserData.DownloadTopCount, IPluginContentProvider.PostsNumberLimit
+        Friend Overridable Property DownloadToDate As Date? = Nothing Implements IUserData.DownloadToDate, IPluginContentProvider.PostsDateLimit
         Protected Responser As PersonalUtilities.Tools.WEB.Response
         Friend Overridable Sub DownloadData(ByVal Token As CancellationToken) Implements IContentProvider.DownloadData
             Dim Canceled As Boolean = False
@@ -791,8 +792,20 @@ BlockNullPicture:
                 If Not Canceled Then _DataParsed = True ': LastUpdated = Now
                 _ContentNew.Clear()
                 DownloadTopCount = Nothing
+                DownloadToDate = Nothing
             End Try
         End Sub
+        Protected Function CheckDatesLimit(ByVal DateString As String, ByVal DateProvider As IFormatProvider) As Boolean
+            Try
+                If DownloadToDate.HasValue And Not DateString.IsEmptyString Then
+                    Dim td As Date? = AConvert(Of Date)(DateString, DateProvider, Nothing)
+                    If td.HasValue Then Return td.Value < DownloadToDate.Value
+                End If
+                Return True
+            Catch ex As Exception
+                Return ErrorsDescriber.Execute(EDP.SendInLog, ex, $"[UserDataBase.CheckDatesLimit({DateString})]", True)
+            End Try
+        End Function
         Protected Sub UpdateDataFiles()
             If Not User.File.IsEmptyString Then
                 MyFileData = User.File
@@ -1236,6 +1249,7 @@ BlockNullPicture:
         Sub OpenFolder()
         ReadOnly Property Self As IUserData
         Property DownloadTopCount As Integer?
+        Property DownloadToDate As Date?
         Sub SetEnvironment(ByRef h As SettingsHost, ByVal u As UserInfo, ByVal _LoadUserInformation As Boolean,
                            Optional ByVal AttachUserInfo As Boolean = True)
         ReadOnly Property Disposed As Boolean

SCrawler/API/Base/UserDataBase.vb

@@ -71,7 +71,7 @@ Namespace API.Instagram
         Friend ReadOnly Property RequestsWaitTimer As PropertyValue
         <Provider(NameOf(RequestsWaitTimer), FieldsChecker:=True)>
         Private ReadOnly Property RequestsWaitTimerProvider As IFormatProvider
-        <PropertyOption(ControlText:="Request timer counter", AllowNull:=False), PXML("RequestsWaitTimerTaskCount"), ControlNumber(6)>
+        <PropertyOption(ControlText:="Request timer counter", AllowNull:=False, LeftOffset:=120), PXML("RequestsWaitTimerTaskCount"), ControlNumber(6)>
         Friend ReadOnly Property RequestsWaitTimerTaskCount As PropertyValue
         <Provider(NameOf(RequestsWaitTimerTaskCount), FieldsChecker:=True)>
         Private ReadOnly Property RequestsWaitTimerTaskCountProvider As IFormatProvider

SCrawler/API/Instagram/SiteSettings.vb

@@ -243,6 +243,7 @@ Namespace API.Instagram
                                                         If Not PostID.IsEmptyString And _TempPostsList.Contains(PostID) Then Throw New ExitException(_DownloadComplete)
                                                         _TempPostsList.Add(PostID)
                                                         PostDate = node.Value("taken_at_timestamp")
+                                                        If Not CheckDatesLimit(PostDate, DateProvider) Then Throw New ExitException(_DownloadComplete)
                                                         ObtainMedia(node, PostID, PostDate, SpecFolder)
                                                     End If
                                                 Next

SCrawler/API/Instagram/UserData.vb

@@ -23,6 +23,11 @@ Namespace API.Reddit
                 Return DirectCast(HOST.Source, SiteSettings)
             End Get
         End Property
+        Private Shared ReadOnly Property DateTrueProvider(ByVal IsChannel As Boolean) As IFormatProvider
+            Get
+                Return If(IsChannel, DateProviderChannel, DateProvider)
+            End Get
+        End Property
 #Region "Channels Support"
 #Region "IChannelLimits Support"
         Friend Property DownloadLimitCount As Integer? Implements IChannelLimits.DownloadLimitCount
@@ -163,6 +168,7 @@ Namespace API.Reddit
                                                 Continue For
                                             End If
                                             If nn.Contains("created") Then PostDate = nn("created").Value Else PostDate = String.Empty
+                                            If DownloadToDate.HasValue AndAlso Not CheckDatesLimit(PostDate, DateTrueProvider(IsChannel)) Then Exit Sub
 
                                             _ItemsBefore = _TempMediaList.Count
                                             added = True
@@ -455,7 +461,7 @@ Namespace API.Reddit
             Dim m As New UserMedia(_URL, t) With {.Post = New UserPost With {.ID = PostID, .UserID = _UserID}}
             If t = UTypes.Picture Or t = UTypes.GIF Then m.File = UrlToFile(m.URL) Else m.File = Nothing
             If m.URL.Contains("preview") Then m.URL = $"https://i.redd.it/{m.File.File}"
-            If Not PostDate.IsEmptyString Then m.Post.Date = AConvert(Of Date)(PostDate, If(IsChannel, DateProviderChannel, DateProvider), Nothing) Else m.Post.Date = Nothing
+            If Not PostDate.IsEmptyString Then m.Post.Date = AConvert(Of Date)(PostDate, DateTrueProvider(IsChannel), Nothing) Else m.Post.Date = Nothing
             Return m
         End Function
         Private Function TryFile(ByVal URL As String) As Boolean

SCrawler/API/Reddit/UserData.vb

@@ -37,6 +37,7 @@ Namespace API.RedGifs
                             pTotal = j.Value("pages").FromXML(Of Integer)(0)
                             For Each g As EContainer In j("gifs")
                                 postDate = g.Value("createDate")
+                                If Not CheckDatesLimit(postDate, DateProvider) Then Exit Sub
                                 postID = g.Value("id")
                                 If Not _TempPostsList.Contains(postID) Then _TempPostsList.Add(postID) Else Exit For
                                 With g("urls")

SCrawler/API/Redgifs/UserData.vb

@@ -64,6 +64,7 @@ Namespace API.Twitter
                                     'Date Pattern:
                                     'Sat Jan 01 01:10:15 +0000 2000
                                     If nn.Contains("created_at") Then PostDate = nn("created_at").Value Else PostDate = String.Empty
+                                    If Not CheckDatesLimit(PostDate, Declarations.DateProvider) Then Exit Sub
 
                                     If Not _TempPostsList.Contains(PostID) Then
                                         NewPostDetected = True

SCrawler/API/Twitter/UserData.vb

@@ -14,6 +14,7 @@ Namespace DownloadObjects
         Private MyView As FormsView
         Friend Property Opened As Boolean = False
         Private ReadOnly JobsList As List(Of DownloadProgress)
+        Friend Property DisableProgressChange As Boolean = False
         Friend Sub New()
             InitializeComponent()
             JobsList = New List(Of DownloadProgress)
@@ -66,7 +67,11 @@ Namespace DownloadObjects
             TP_MAIN.Refresh()
         End Sub
         Private Sub Jobs_OnTotalCountChange()
-            If JobsList.Count > 0 Then MainProgress.TotalCount = JobsList.Sum(Function(j) CLng(j.Job.Progress.TotalCount))
+            If JobsList.Count > 0 And Not DisableProgressChange Then
+                MainProgress.TotalCount = JobsList.Sum(Function(j) CLng(j.Job.Progress.TotalCount))
+                MainProgress.CurrentCounter = Math.Max(JobsList.Sum(Function(j) CLng(j.Job.Progress.CurrentCounter)) - 1, 0)
+                If MainProgress.CurrentCounter > 0 Then MainProgress.Perform()
+            End If
         End Sub
     End Class
 End Namespace

SCrawler/Download/ActiveDownloadingProgress.vb

@@ -217,7 +217,9 @@ Namespace DownloadObjects
         Private Sub JobsChecker()
             Try
                 MainProgress.TotalCount = 0
-                Do While Pool.Exists(Function(p) p.Count > 0)
+                MainProgress.CurrentCounter = 0
+                MyProgressForm.DisableProgressChange = False
+                Do While Pool.Exists(Function(p) p.Count > 0 Or p.Working)
                     For Each j As Job In Pool
                         If j.Count > 0 And Not j.Working Then j.Start(New ThreadStart(Sub() StartDownloading(j)))
                     Next
@@ -231,6 +233,7 @@ Namespace DownloadObjects
                     .InformationTemporary = "All data downloaded"
                     .Enabled(EOptions.ProgressBar) = False
                 End With
+                MyProgressForm.DisableProgressChange = True
                 If Pool.Count > 0 Then Pool.ForEach(Sub(p) If Not p.Progress Is Nothing Then p.Progress.TotalCount = 0)
             End Try
         End Sub