Skip to content

Commit 5d5ce29

Browse files
AAndyProgramAAndyProgram
committed
2.0.0.3
Added GetUserMediaOnly for Reddit Fixed Reddit CrossPosts issue Fixed validating new users (collections) Fixed incorrect count of Instagram download tasks
1 parent 2a2fc45 commit 5d5ce29

10 files changed

+115
-90
lines changed

‎Changelog.md

+15
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,18 @@
1+
# 2.0.0.3
2+
3+
**Removed compatibility of program settings with version 1.0.0.4 and lower.**
4+
5+
**If your program version is 1.0.0.4 and lower, it is strongly recommended that you upgrade to release 2.0.0.2 to update the program settings (and run the program). Then update to this release. Otherwise, you will have to configure the program settings again**
6+
7+
**If your program version is 1.0.1.0 or higher, you should not pay attention to this message.**
8+
9+
- Added
10+
- The "Get User Media Only" setting is now available for Reddit. If checked then "CrossPosts" will be skipped, otherwise "CrossPosts" will be included.
11+
- Fixed
12+
- In some cases, the program did not parse all Reddit posts.
13+
- Collection ignored when validated when creating a new user
14+
- Incorrect number of Instagram profiles downloads per session
15+
116
# 2.0.0.2
217

318
**This is the last release that supports program settings of version 1.0.0.4 and lower. Compatibility of program settings with version 1.0.0.4 and lower will be removed in future releases. It is strongly recommended that you upgrade to this release before future releases. Otherwise, you will have to configure the program settings again. If your program version is 1.0.1.0 or higher, you should not pay attention to this message.**

‎SCrawler/API/Base/SiteSettings.vb

+1-7
Original file line numberDiff line numberDiff line change
@@ -123,7 +123,6 @@ Namespace API.Base
123123

124124
Dim n() As String = {SettingsCLS.Name_Node_Sites, Site.ToString}
125125
_Path = New XMLValue(Of SFile)("Path", SFile.GetPath($"{GlobalPath.PathWithSeparator}{Site}"), _XML, n, XMLValue(Of SFile).ToFilePath)
126-
_Path.ReplaceByValue("Path", {Site.ToString})
127126
_XML.Remove(Site.ToString)
128127

129128
Temporary = New XMLValue(Of Boolean)
@@ -138,12 +137,7 @@ Namespace API.Base
138137
DownloadVideos.SetExtended("DownloadVideos", True, _XML, n)
139138
DownloadVideos.SetDefault(_Vids)
140139

141-
If Site = Sites.Twitter Then
142-
GetUserMediaOnly = New XMLValue(Of Boolean)("GetUserMediaOnly", True, _XML, n)
143-
GetUserMediaOnly.ReplaceByValue("TwitterDefaultGetUserMedia", n)
144-
Else
145-
GetUserMediaOnly = New XMLValue(Of Boolean)
146-
End If
140+
GetUserMediaOnly = New XMLValue(Of Boolean)("GetUserMediaOnly", True, _XML, n)
147141

148142
CreateProp(InstaHashUpdateRequired, Sites.Instagram, "InstaHashUpdateRequired", True, _XML, n)
149143
CreateProp(InstaHash, Sites.Instagram, "InstaHash", String.Empty, _XML, n)

‎SCrawler/API/Reddit/UserData.vb

+39-20
Original file line numberDiff line numberDiff line change
@@ -54,12 +54,14 @@ Namespace API.Reddit
5454
Private Sub New()
5555
ChannelPostsNames = New List(Of String)
5656
_ExistsUsersNames = New List(Of String)
57+
_CrossPosts = New List(Of String)
5758
End Sub
5859
''' <summary>Default initializer</summary>
5960
Friend Sub New(ByVal u As UserInfo, Optional ByVal _LoadUserInformation As Boolean = True, Optional ByVal InvokeImageHandler As Boolean = True)
6061
MyBase.New(InvokeImageHandler)
6162
ChannelPostsNames = New List(Of String)
6263
_ExistsUsersNames = New List(Of String)
64+
_CrossPosts = New List(Of String)
6365
User = u
6466
If _LoadUserInformation Then LoadUserInformation()
6567
End Sub
@@ -108,10 +110,13 @@ Namespace API.Reddit
108110
#End Region
109111
#Region "Download Functions (User, Channel)"
110112
Private _TotalPostsDownloaded As Integer = 0
113+
Private ReadOnly _CrossPosts As List(Of String)
111114
Private Sub DownloadDataUser(ByVal POST As String, ByVal Token As CancellationToken)
115+
Const CPRI$ = "crosspostRootId"
116+
Const CPPI$ = "crosspostParentId"
112117
Dim URL$ = String.Empty
113118
Try
114-
Dim PostID$ = String.Empty
119+
Dim PostID$ = String.Empty, PostTmp$ = String.Empty
115120
Dim PostDate$
116121
Dim n As EContainer, nn As EContainer, s As EContainer
117122
Dim NewPostDetected As Boolean = False
@@ -120,8 +125,10 @@ Namespace API.Reddit
120125
Dim added As Boolean
121126
Dim __ItemType$
122127
Dim tmpType As UTypes
123-
Dim CheckNode As Predicate(Of EContainer) = Function(e) e("author").XmlIfNothingValue("/").ToLower.Equals(Name.ToLower)
128+
Dim IsCrossPost As Predicate(Of EContainer) = Function(e) Not (e.Value(CPRI).IsEmptyString And e.Value(CPPI).IsEmptyString)
129+
Dim CheckNode As Predicate(Of EContainer) = Function(e) Not ParseUserMediaOnly OrElse e("author").XmlIfNothingValue("/").ToLower.Equals(Name.ToLower)
124130
Dim UPicType As Func(Of String, UTypes) = Function(input) IIf(input = "image", UTypes.Picture, UTypes.GIF)
131+
Dim _PostID As Func(Of String) = Function() IIf(PostTmp.IsEmptyString, PostID, PostTmp)
125132

126133
URL = $"https://gateway.reddit.com/desktopapi/v1/user/{Name}/posts?rtj=only&allow_quarantined=true&allow_over18=1&include=identity&after={POST}&dist=25&sort=new&t=all&layout=classic"
127134
ThrowAny(Token)
@@ -134,39 +141,51 @@ Namespace API.Reddit
134141
For Each nn In n
135142
ThrowAny(Token)
136143
If nn.Count > 0 Then
137-
PostID = nn.Name
138-
If PostID.IsEmptyString AndAlso nn.Contains("id") Then PostID = nn("id").Value
139-
If nn.Contains("created") Then PostDate = nn("created").Value Else PostDate = String.Empty
140-
If Not _TempPostsList.Contains(PostID) Then
141-
NewPostDetected = True
142-
_TempPostsList.Add(PostID)
143-
Else
144-
ExistsDetected = True
145-
Continue For
146-
End If
147-
148144
If CheckNode(nn) Then
145+
146+
'Obtain post ID
147+
PostTmp = nn.Name
148+
If PostTmp.IsEmptyString Then PostTmp = nn.Value("id")
149+
If PostTmp.IsEmptyString Then Continue For
150+
'Check for CrossPost
151+
If IsCrossPost(nn) Then
152+
_CrossPosts.ListAddList({nn.Value(CPRI), nn.Value(CPPI)}, LNC)
153+
Continue For
154+
Else
155+
If Not _CrossPosts.Contains(PostTmp) Then PostID = PostTmp : PostTmp = String.Empty
156+
End If
157+
158+
'Download decision
159+
If Not _TempPostsList.Contains(_PostID()) Then
160+
NewPostDetected = True
161+
_TempPostsList.Add(_PostID())
162+
Else
163+
If Not _CrossPosts.Contains(_PostID()) Then ExistsDetected = True
164+
Continue For
165+
End If
166+
If nn.Contains("created") Then PostDate = nn("created").Value Else PostDate = String.Empty
167+
149168
_ItemsBefore = _TempMediaList.Count
150169
added = True
151170
s = nn.ItemF({"source", "url"})
152171
If s.XmlIfNothingValue("/").Contains("redgifs.com") Then
153-
_TempMediaList.ListAddValue(MediaFromData(UTypes.VideoPre, s.Value, PostID, PostDate,, IsChannel), LNC)
154-
ElseIf Not CreateImgurMedia(s.XmlIfNothingValue, PostID, PostDate,, IsChannel) Then
172+
_TempMediaList.ListAddValue(MediaFromData(UTypes.VideoPre, s.Value, _PostID(), PostDate,, IsChannel), LNC)
173+
ElseIf Not CreateImgurMedia(s.XmlIfNothingValue, _PostID(), PostDate,, IsChannel) Then
155174
s = nn.ItemF({"media"}).XmlIfNothing
156175
__ItemType = s("type").XmlIfNothingValue
157176
Select Case __ItemType
158-
Case "gallery" : If Not DownloadGallery(s, PostID, PostDate) Then added = False
177+
Case "gallery" : If Not DownloadGallery(s, _PostID(), PostDate) Then added = False
159178
Case "image", "gifvideo"
160179
If s.Contains("content") Then
161180
_TempMediaList.ListAddValue(MediaFromData(UPicType(__ItemType), s.Value("content"),
162-
PostID, PostDate,, IsChannel), LNC)
181+
_PostID(), PostDate,, IsChannel), LNC)
163182
Else
164183
added = False
165184
End If
166185
Case "video"
167186
If Settings.UseM3U8 AndAlso s("hlsUrl").XmlIfNothingValue("/").ToLower.Contains("m3u8") Then
168187
_TempMediaList.ListAddValue(MediaFromData(UTypes.m3u8, s.Value("hlsUrl"),
169-
PostID, PostDate,, IsChannel), LNC)
188+
_PostID(), PostDate,, IsChannel), LNC)
170189
Else
171190
added = False
172191
End If
@@ -186,7 +205,7 @@ Namespace API.Reddit
186205
End Select
187206
End With
188207
If Not tmpType = UTypes.Undefined Then
189-
_TempMediaList.ListAddValue(MediaFromData(tmpType, s.Value, PostID, PostDate,, IsChannel), LNC)
208+
_TempMediaList.ListAddValue(MediaFromData(tmpType, s.Value, _PostID(), PostDate,, IsChannel), LNC)
190209
End If
191210
End If
192211
End If
@@ -596,7 +615,7 @@ Namespace API.Reddit
596615
Return 1
597616
End Function
598617
Protected Overrides Sub Dispose(ByVal disposing As Boolean)
599-
If Not disposedValue And disposing Then ChannelPostsNames.Clear() : _ExistsUsersNames.Clear()
618+
If Not disposedValue And disposing Then ChannelPostsNames.Clear() : _ExistsUsersNames.Clear() : _CrossPosts.Clear()
600619
MyBase.Dispose(disposing)
601620
End Sub
602621
End Class

0 commit comments

Comments
 (0)