Skip to content

Commit 55a28a7

Browse files
AAndyProgramAAndyProgram
committed
1.0.0.2
Add: downloading certain types of media, name files by date Fix: Stop button is not activated after download start
1 parent c7dc0ec commit 55a28a7

16 files changed

+597
-124
lines changed

‎Changelog.md

+8
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,11 @@
1+
# 1.0.0.2
2+
3+
- Added
4+
- Ability to choose what types of media you want to download (images only, videos only, both)
5+
- Ability to name files by date
6+
- Fixed
7+
- In some cases, the "Stop" button is not activated after download start
8+
19
# 1.0.0.1
210

311
- Added

‎SCrawler/API/Base/Structures.vb

+1
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,7 @@
4949
''' <summary>Post ID</summary>
5050
Friend ID As String
5151
Friend [Date] As Date?
52+
Friend Title As String
5253
#Region "Channel compatible fields"
5354
Friend UserID As String
5455
Friend CachedFile As SFile

‎SCrawler/API/Base/UserDataBase.vb

+39
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,8 @@ Namespace API.Base
4747
Private Const Name_LabelsName As String = "Labels"
4848

4949
Private Const Name_ReadyForDownload As String = "ReadyForDownload"
50+
Private Const Name_DownloadImages As String = "DownloadImages"
51+
Private Const Name_DownloadVideos As String = "DownloadVideos"
5052

5153
Private Const Name_VideoCount As String = "VideoCount"
5254
Private Const Name_PicturesCount As String = "PicturesCount"
@@ -233,6 +235,8 @@ BlockNullPicture:
233235
Protected _DataLoaded As Boolean = False
234236
Protected _DataParsed As Boolean = False
235237
Friend Property ReadyForDownload As Boolean = True Implements IUserData.ReadyForDownload
238+
Friend Property DownloadImages As Boolean = True Implements IUserData.DownloadImages
239+
Friend Property DownloadVideos As Boolean = True Implements IUserData.DownloadVideos
236240
#End Region
237241
#Region "Content"
238242
Protected ReadOnly _ContentList As List(Of UserMedia)
@@ -431,6 +435,8 @@ BlockNullPicture:
431435
CreatedByChannel = x.Value(Name_CreatedByChannel).FromXML(Of Boolean)(False)
432436
SeparateVideoFolder = AConvert(Of Boolean)(x.Value(Name_SeparateVideoFolder), Nothing)
433437
ReadyForDownload = x.Value(Name_ReadyForDownload).FromXML(Of Boolean)(True)
438+
DownloadImages = x.Value(Name_DownloadImages).FromXML(Of Boolean)(True)
439+
DownloadedVideos = x.Value(Name_DownloadVideos).FromXML(Of Boolean)(True)
434440
_CountVideo = x.Value(Name_VideoCount).FromXML(Of Integer)(0)
435441
_CountPictures = x.Value(Name_PicturesCount).FromXML(Of Integer)(0)
436442
LastUpdated = AConvert(Of Date)(x.Value(Name_LastUpdated), ADateTime.Formats.BaseDateTime, Nothing)
@@ -463,6 +469,8 @@ BlockNullPicture:
463469
x.Add(Name_SeparateVideoFolder, String.Empty)
464470
End If
465471
x.Add(Name_ReadyForDownload, ReadyForDownload.BoolToInteger)
472+
x.Add(Name_DownloadImages, DownloadImages.BoolToInteger)
473+
x.Add(Name_DownloadVideos, DownloadVideos.BoolToInteger)
466474
x.Add(Name_VideoCount, _CountVideo)
467475
x.Add(Name_PicturesCount, _CountPictures)
468476
x.Add(Name_LastUpdated, AConvert(Of String)(LastUpdated, ADateTime.Formats.BaseDateTime, String.Empty))
@@ -586,6 +594,12 @@ BlockNullPicture:
586594
DownloadDataF(Token)
587595
ThrowAny(Token)
588596

597+
If _TempMediaList.Count > 0 Then
598+
If Not DownloadImages Then _TempMediaList.RemoveAll(Function(m) m.Type = UserMedia.Types.GIF Or m.Type = UserMedia.Types.Picture)
599+
If Not DownloadVideos Then _TempMediaList.RemoveAll(Function(m) m.Type = UserMedia.Types.Video Or
600+
m.Type = UserMedia.Types.VideoPre Or m.Type = UserMedia.Types.m3u8)
601+
End If
602+
589603
ReparseVideo(Token)
590604
ThrowAny(Token)
591605
If _TempPostsList.Count > 0 And __SaveData Then TextSaver.SaveTextToFile(_TempPostsList.ListToString(, Environment.NewLine), MyFilePosts, True,, EDP.None)
@@ -640,6 +654,29 @@ BlockNullPicture:
640654
Protected MustOverride Sub DownloadDataF(ByVal Token As CancellationToken)
641655
Protected MustOverride Sub ReparseVideo(ByVal Token As CancellationToken)
642656
Protected MustOverride Sub DownloadContent(ByVal Token As CancellationToken)
657+
Protected Function ChangeFileNameByProvider(ByVal f As SFile, ByVal m As UserMedia) As SFile
658+
Dim ff As SFile = Nothing
659+
Try
660+
If Not f.IsEmptyString AndAlso f.Exists Then
661+
Dim d As Date? = m.Post.Date
662+
If Settings.FileReplaceNameByDate Then
663+
Dim dd$ = AConvert(Of String)(If(d, Now), FileDateAppenderProvider, String.Empty)
664+
ff = f
665+
ff.Name = dd
666+
ff = SFile.Indexed_IndexFile(ff,, New NumberedFile(ff))
667+
ElseIf d.HasValue AndAlso (Settings.FileAddDateToFileName Or Settings.FileAddTimeToFileName) AndAlso
668+
(Not FileDateAppenderProvider Is Nothing And Not FileDateAppenderPattern.IsEmptyString) Then
669+
ff = f
670+
ff.Name = String.Format(FileDateAppenderPattern, f.Name, CStr(AConvert(Of String)(d.Value, FileDateAppenderProvider, String.Empty)))
671+
End If
672+
If Not ff.Name.IsEmptyString Then My.Computer.FileSystem.RenameFile(f, ff.File) : Return ff
673+
End If
674+
Return f
675+
Catch ex As Exception
676+
LogError(ex, $"change file name from [{f}] to [{ff}]")
677+
Return f
678+
End Try
679+
End Function
643680
#End Region
644681
#Region "Delete, Move, Merge"
645682
Friend Overridable Function Delete() As Integer Implements IUserData.Delete
@@ -916,6 +953,8 @@ BlockNullPicture:
916953
ReadOnly Property FitToAddParams As Boolean
917954
ReadOnly Property LVIKey As String
918955
ReadOnly Property LVIIndex As Integer
956+
Property DownloadImages As Boolean
957+
Property DownloadVideos As Boolean
919958
Function GetLVI(ByVal Destination As ListView) As ListViewItem
920959
Function GetLVIGroup(ByVal Destination As ListView) As ListViewGroup
921960
Sub LoadUserInformation()

‎SCrawler/API/Reddit/UserData.vb

+30-17
Original file line numberDiff line numberDiff line change
@@ -92,7 +92,7 @@ Namespace API.Reddit
9292
Dim URL$ = String.Empty
9393
Try
9494
Dim PostID$ = String.Empty
95-
Dim PostDate$
95+
Dim PostDate$, PostTitle$
9696
Dim n As EContainer, nn As EContainer, s As EContainer
9797
Dim NewPostDetected As Boolean = False
9898
Dim ExistsDetected As Boolean = False
@@ -107,7 +107,7 @@ Namespace API.Reddit
107107
ThrowAny(Token)
108108
Dim r$ = GetSiteResponse(URL)
109109
If Not r.IsEmptyString Then
110-
Using w As EContainer = JsonDocument.Parse(r)
110+
Using w As EContainer = JsonDocument.Parse(r).XmlIfNothing
111111
If w.Count > 0 Then
112112
n = w.GetNode(JsonNodesJson)
113113
If Not n Is Nothing AndAlso n.Count > 0 Then
@@ -124,29 +124,32 @@ Namespace API.Reddit
124124
ExistsDetected = True
125125
Continue For
126126
End If
127+
PostTitle = nn.Value("title")
128+
127129
If CheckNode(nn) Then
128130
_ItemsBefore = _TempMediaList.Count
129131
added = True
130132
s = nn.ItemF({"source", "url"})
131133
If s.XmlIfNothingValue("/").Contains("redgifs.com") Then
132-
_TempMediaList.ListAddValue(MediaFromData(UTypes.VideoPre, s.Value, PostID, PostDate,, IsChannel), LNC)
133-
_TotalPostsDownloaded += 1
134+
_TempMediaList.ListAddValue(MediaFromData(UTypes.VideoPre, s.Value, PostID, PostDate,, IsChannel, PostTitle), LNC)
134135
Else
135136
s = nn.ItemF({"media"}).XmlIfNothing
136137
__ItemType = s("type").XmlIfNothingValue
137138
Select Case __ItemType
138-
Case "gallery" : DownloadGallery(s, PostID, PostDate) : _TotalPostsDownloaded += 1
139+
Case "gallery" : If Not DownloadGallery(s, PostID, PostDate,,, PostTitle) Then added = False
139140
Case "image", "gifvideo"
140141
If s.Contains("content") Then
141142
_TempMediaList.ListAddValue(MediaFromData(UPicType(__ItemType), s.Value("content"),
142-
PostID, PostDate,, IsChannel), LNC)
143-
_TotalPostsDownloaded += 1
143+
PostID, PostDate,, IsChannel, PostTitle), LNC)
144+
Else
145+
added = False
144146
End If
145147
Case "video"
146148
If Settings.UseM3U8 AndAlso s("hlsUrl").XmlIfNothingValue("/").ToLower.Contains("m3u8") Then
147149
_TempMediaList.ListAddValue(MediaFromData(UTypes.m3u8, s.Value("hlsUrl"),
148-
PostID, PostDate,, IsChannel), LNC)
149-
_TotalPostsDownloaded += 1
150+
PostID, PostDate,, IsChannel, PostTitle), LNC)
151+
Else
152+
added = False
150153
End If
151154
Case Else : added = False
152155
End Select
@@ -164,8 +167,7 @@ Namespace API.Reddit
164167
End Select
165168
End With
166169
If Not tmpType = UTypes.Undefined Then
167-
_TempMediaList.ListAddValue(MediaFromData(tmpType, s.Value, PostID, PostDate,, IsChannel), LNC)
168-
_TotalPostsDownloaded += 1
170+
_TempMediaList.ListAddValue(MediaFromData(tmpType, s.Value, PostID, PostDate,, IsChannel, PostTitle), LNC)
169171
End If
170172
End If
171173
End If
@@ -267,25 +269,30 @@ Namespace API.Reddit
267269
End Sub
268270
#End Region
269271
#Region "Download Base Functions"
270-
Private Sub DownloadGallery(ByVal w As EContainer, ByVal PostID As String, ByVal PostDate As String,
271-
Optional ByVal _UserID As String = Nothing, Optional ByVal FirstOnly As Boolean = False)
272+
Private Function DownloadGallery(ByVal w As EContainer, ByVal PostID As String, ByVal PostDate As String,
273+
Optional ByVal _UserID As String = Nothing, Optional ByVal FirstOnly As Boolean = False,
274+
Optional ByVal Title As String = Nothing) As Boolean
272275
Try
276+
Dim added As Boolean = False
273277
Dim cn$ = IIf(IsChannel, "media_metadata", "mediaMetadata")
274278
If Not w Is Nothing AndAlso w(cn).XmlIfNothing.Count > 0 Then
275279
Dim t As EContainer
276280
For Each n As EContainer In w(cn)
277281
t = n.ItemF({"s", "u"})
278282
If Not t Is Nothing AndAlso Not t.Value.IsEmptyString Then
279-
_TempMediaList.ListAddValue(MediaFromData(UTypes.Picture, t.Value, PostID, PostDate, _UserID, IsChannel), LNC)
283+
_TempMediaList.ListAddValue(MediaFromData(UTypes.Picture, t.Value, PostID, PostDate, _UserID, IsChannel, Title), LNC)
284+
added = True
280285
If FirstOnly Then Exit For
281286
End If
282287
Next
283288
End If
289+
Return added
284290
Catch ex As Exception
285291
LogError(ex, "gallery parsing error")
286292
HasError = True
293+
Return False
287294
End Try
288-
End Sub
295+
End Function
289296
Protected Overrides Sub ReparseVideo(ByVal Token As CancellationToken)
290297
Try
291298
ThrowAny(Token)
@@ -333,13 +340,15 @@ Namespace API.Reddit
333340
#End Region
334341
#Region "Structure creator"
335342
Protected Shared Function MediaFromData(ByVal t As UTypes, ByVal _URL As String, ByVal PostID As String, ByVal PostDate As String,
336-
Optional ByVal _UserID As String = "", Optional ByVal IsChannel As Boolean = False) As UserMedia
343+
Optional ByVal _UserID As String = "", Optional ByVal IsChannel As Boolean = False,
344+
Optional ByVal Title As String = Nothing) As UserMedia
337345
If _URL.IsEmptyString And t = UTypes.Picture Then Return Nothing
338346
_URL = LinkFormatterSecure(RegexReplace(_URL.Replace("\", String.Empty), LinkPattern))
339347
Dim m As New UserMedia(_URL, t) With {.Post = New UserPost With {.ID = PostID, .UserID = _UserID}}
340348
If t = UTypes.Picture Or t = UTypes.GIF Then m.File = UrlToFile(m.URL) Else m.File = Nothing
341349
If m.URL.Contains("preview") Then m.URL = $"https://i.redd.it/{m.File.File}"
342350
If Not PostDate.IsEmptyString Then m.Post.Date = AConvert(Of Date)(PostDate, If(IsChannel, DateProviderChannel, DateProvider), Nothing) Else m.Post.Date = Nothing
351+
If Not Title.IsEmptyString Then m.Post.Title = Title
343352
Return m
344353
End Function
345354
Private Function TryFile(ByVal URL As String) As Boolean
@@ -436,7 +445,11 @@ Namespace API.Reddit
436445
Case UTypes.Picture : DownloadedPictures += 1 : _CountPictures += 1
437446
Case UTypes.Video, UTypes.m3u8 : DownloadedVideos += 1 : _CountVideo += 1
438447
End Select
439-
v.File = f
448+
If Not IsChannel Or Not SaveToCache Then
449+
v.File = ChangeFileNameByProvider(f, v)
450+
Else
451+
v.File = f
452+
End If
440453
v.Post.CachedFile = f
441454
v.State = UStates.Downloaded
442455
dCount += 1

0 commit comments

Comments
 (0)